public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v9 3/4] Remove globals readOff, readLen and readSegNo 58+ messages / 7 participants [nested] [flat]
* [PATCH v9 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e380eaa186..1ef6574f5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -800,18 +800,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -901,10 +897,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7638,7 +7636,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11802,13 +11801,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11821,7 +11821,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11829,10 +11829,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11842,7 +11842,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11851,17 +11851,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11889,40 +11886,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11950,15 +11943,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -11967,14 +11960,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12006,7 +11998,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index e59e42bee3..a862db6d90 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -135,6 +135,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.2 ----Next_Part(Wed_Apr_22_10_12_46_2020_740)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v9-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v10 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 840218efee..694307a177 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -801,18 +801,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -902,10 +898,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7662,7 +7660,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11844,13 +11843,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11863,7 +11863,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11871,10 +11871,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11884,7 +11884,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11893,17 +11893,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11931,40 +11928,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11992,15 +11985,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12009,14 +12002,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12048,7 +12040,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 1d0167e26d..79bb2bc537 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.2 ----Next_Part(Tue_May_26_16_40_02_2020_373)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v10-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v13 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 9d5d71ac8c..ce62559f42 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -782,14 +782,10 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* @@ -876,10 +872,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, int source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7526,7 +7524,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11578,13 +11577,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11597,7 +11597,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11605,10 +11605,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11618,7 +11618,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = 0; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11627,17 +11627,14 @@ retry: receivedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = 0; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11665,40 +11662,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11726,15 +11719,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -11743,14 +11736,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = 0; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -11782,7 +11774,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index e59e42bee3..a862db6d90 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -135,6 +135,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.2 ----Next_Part(Tue_Jan_28_21_20_20_2020_871)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v13-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v8 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e380eaa186..1ef6574f5f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -800,18 +800,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -901,10 +897,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7638,7 +7636,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11802,13 +11801,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11821,7 +11821,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11829,10 +11829,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11842,7 +11842,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11851,17 +11851,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11889,40 +11886,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11950,15 +11943,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -11967,14 +11960,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12006,7 +11998,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index e59e42bee3..a862db6d90 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -135,6 +135,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.2 ----Next_Part(Tue_Apr_21_17_04_27_2020_275)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v8-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v19 3/5] Remove globals readOff, readLen and readSegNo. @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two global variables are duplicated in XLogReaderState. Remove them, and also readSegNo, which should move into that struct too. Author: Kyotaro HORIGUCHI <[email protected]> Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b7d7e6d31b..9a9835f05d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -811,17 +811,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -913,10 +909,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7808,7 +7806,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -12097,13 +12096,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -12116,7 +12116,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12124,10 +12124,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12137,7 +12137,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12146,17 +12146,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->seg.ws_segno)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12184,40 +12181,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->seg.ws_segno, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->seg.ws_segno); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12245,15 +12238,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12262,14 +12255,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12301,7 +12293,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; -- 2.27.0 ----Next_Part(Wed_Apr__7_17_50_25_2021_942)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v19-0004-Make-XLogFindNextRecord-not-use-callback-functio.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v16 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 6ef1f817ad..eccb88b611 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -801,17 +801,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -902,10 +898,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7662,7 +7660,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11856,13 +11855,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11875,7 +11875,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11883,10 +11883,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11896,7 +11896,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11905,17 +11905,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11943,40 +11940,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12004,15 +11997,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12021,14 +12014,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12060,7 +12052,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 38f9e59f99..732d463a49 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Tue_Sep__8_16_35_16_2020_701)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v16-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 79 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f9b0108602..06e7ff4a24 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -804,18 +804,14 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; -static XLogSource readSource = XLOG_FROM_ANY; +static XLogSource readSource = 0; /* XLOG_FROM_* code */ /* * Keeps track of which source we're currently reading from. This is @@ -905,10 +901,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7665,7 +7663,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11850,13 +11849,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11869,7 +11869,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -11877,10 +11877,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -11890,7 +11890,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -11899,17 +11899,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -11937,40 +11934,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -11998,15 +11991,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12015,14 +12008,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12054,7 +12046,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 6a7fe140cb..09fc692fa1 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.18.4 ----Next_Part(Thu_Jul__2_13_53_30_2020_072)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v15-0004-Change-policy-of-XLog-read-buffer-allocation.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo @ 2019-09-10 08:28 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2019-09-10 08:28 UTC (permalink / raw) The first two variables are functionally duplicate with them in XLogReaderState. Remove the globals along with readSegNo, which behaves in the similar way. --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- src/include/access/xlogreader.h | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670996ae67..b517ca85ad 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; * These variables are used similarly to the ones above, but for reading * the XLOG. Note, however, that readOff generally represents the offset * of the page just read, not the seek position of the FD itself, which - * will be just past that page. readLen indicates how much of the current - * page has been read into readBuf, and readSource indicates where we got - * the currently open file from. + * will be just past that page. readSource indicates where we got the + * currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -910,10 +906,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); static void PreallocXlogFiles(XLogRecPtr endptr); @@ -7730,7 +7728,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -11977,13 +11976,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -11996,7 +11996,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->readSegNo, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12004,10 +12004,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (bgwriterLaunched) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->readSegNo)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12017,7 +12017,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->readSegNo, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12026,17 +12026,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->readSegNo)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12064,40 +12061,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->readSegNo, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->readSegNo); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12125,15 +12118,15 @@ retry: * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (!XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { - /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + /* reset any error StateValidatePageHeader() might have set */ + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12142,14 +12135,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12181,7 +12173,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 4e9268b553..b9dad7ee51 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -156,6 +156,7 @@ struct XLogReaderState * read by reader, which must be larger than * the request, or -1 on error */ TimeLineID readPageTLI; /* TLI for data currently in readBuf */ + XLogSegNo readSegNo; /* Segment # for data currently in readBuf */ char *readBuf; /* buffer to store data */ bool page_verified; /* is the page header on the buffer verified? */ bool record_verified;/* is the current record header verified? */ -- 2.27.0 ----Next_Part(Thu_Mar__4_11_28_53_2021_014)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v17-0004-Allow-xlogreader-to-use-different-xlog-blocksize.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* [PATCH v18 3/5] Remove globals readOff, readLen and readSegNo. @ 2021-09-30 04:16 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Kyotaro Horiguchi @ 2021-09-30 04:16 UTC (permalink / raw) The first two global variables are duplicated in XLogReaderState. Remove them, and also readSegNo, which should move into that struct too. Author: Kyotaro HORIGUCHI <[email protected]> Discussion: https://postgr.es/m/20190418.210257.43726183.horiguchi.kyotaro%40lab.ntt.co.jp --- src/backend/access/transam/xlog.c | 77 ++++++++++++++----------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 316ba256f6..c4fe006776 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -809,17 +809,13 @@ static XLogSegNo openLogSegNo = 0; /* * These variables are used similarly to the ones above, but for reading - * the XLOG. readOff is the offset of the page just read, readLen - * indicates how much of it has been read into readBuf, and readSource + * the XLOG. readOff is the offset of the page just read, readSource * indicates where we got the currently open file from. * Note: we could use Reserve/ReleaseExternalFD to track consumption of * this FD too; but it doesn't currently seem worthwhile, since the XLOG is * not read by general-purpose sessions. */ static int readFile = -1; -static XLogSegNo readSegNo = 0; -static uint32 readOff = 0; -static uint32 readLen = 0; static XLogSource readSource = XLOG_FROM_ANY; /* @@ -911,10 +907,12 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath, static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, XLogSource source, bool notfoundOk); static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source); -static bool XLogPageRead(XLogReaderState *xlogreader, +static bool XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess); static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr); + bool fetching_ckpt, + XLogRecPtr tliRecPtr, + XLogSegNo readSegNo); static void XLogShutdownWalRcv(void); static int emode_for_corrupt_record(int emode, XLogRecPtr RecPtr); static void XLogFileClose(void); @@ -7894,7 +7892,8 @@ StartupXLOG(void) XLogRecPtr pageBeginPtr; pageBeginPtr = EndOfLog - (EndOfLog % XLOG_BLCKSZ); - Assert(readOff == XLogSegmentOffset(pageBeginPtr, wal_segment_size)); + Assert(XLogSegmentOffset(xlogreader->readPagePtr, wal_segment_size) == + XLogSegmentOffset(pageBeginPtr, wal_segment_size)); firstIdx = XLogRecPtrToBufIdx(EndOfLog); @@ -12292,13 +12291,14 @@ CancelBackup(void) * sleep and retry. */ static bool -XLogPageRead(XLogReaderState *xlogreader, +XLogPageRead(XLogReaderState *state, bool fetching_ckpt, int emode, bool randAccess) { - char *readBuf = xlogreader->readBuf; - XLogRecPtr targetPagePtr = xlogreader->readPagePtr; - int reqLen = xlogreader->readLen; - XLogRecPtr targetRecPtr = xlogreader->ReadRecPtr; + char *readBuf = state->readBuf; + XLogRecPtr targetPagePtr = state->readPagePtr; + int reqLen = state->readLen; + int readLen = 0; + XLogRecPtr targetRecPtr = state->ReadRecPtr; uint32 targetPageOff; XLogSegNo targetSegNo PG_USED_FOR_ASSERTS_ONLY; int r; @@ -12311,7 +12311,7 @@ XLogPageRead(XLogReaderState *xlogreader, * is not in the currently open one. */ if (readFile >= 0 && - !XLByteInSeg(targetPagePtr, readSegNo, wal_segment_size)) + !XLByteInSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size)) { /* * Request a restartpoint if we've replayed too much xlog since the @@ -12319,10 +12319,10 @@ XLogPageRead(XLogReaderState *xlogreader, */ if (ArchiveRecoveryRequested && IsUnderPostmaster) { - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) { (void) GetRedoRecPtr(); - if (XLogCheckpointNeeded(readSegNo)) + if (XLogCheckpointNeeded(state->seg.ws_segno)) RequestCheckpoint(CHECKPOINT_CAUSE_XLOG); } } @@ -12332,7 +12332,7 @@ XLogPageRead(XLogReaderState *xlogreader, readSource = XLOG_FROM_ANY; } - XLByteToSeg(targetPagePtr, readSegNo, wal_segment_size); + XLByteToSeg(targetPagePtr, state->seg.ws_segno, wal_segment_size); retry: /* See if we need to retrieve more data */ @@ -12341,17 +12341,14 @@ retry: flushedUpto < targetPagePtr + reqLen)) { if (!WaitForWALToBecomeAvailable(targetPagePtr + reqLen, - randAccess, - fetching_ckpt, - targetRecPtr)) + randAccess, fetching_ckpt, + targetRecPtr, state->seg.ws_segno)) { if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; - - xlogreader->readLen = -1; + state->readLen = -1; return false; } } @@ -12379,40 +12376,36 @@ retry: else readLen = XLOG_BLCKSZ; - /* Read the requested page */ - readOff = targetPageOff; - pgstat_report_wait_start(WAIT_EVENT_WAL_READ); - r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) readOff); + r = pg_pread(readFile, readBuf, XLOG_BLCKSZ, (off_t) targetPageOff); if (r != XLOG_BLCKSZ) { char fname[MAXFNAMELEN]; int save_errno = errno; pgstat_report_wait_end(); - XLogFileName(fname, curFileTLI, readSegNo, wal_segment_size); + XLogFileName(fname, curFileTLI, state->seg.ws_segno, wal_segment_size); if (r < 0) { errno = save_errno; ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u: %m", - fname, readOff))); + fname, targetPageOff))); } else ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen), (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read from log segment %s, offset %u: read %d of %zu", - fname, readOff, r, (Size) XLOG_BLCKSZ))); + fname, targetPageOff, r, (Size) XLOG_BLCKSZ))); goto next_record_is_invalid; } pgstat_report_wait_end(); - Assert(targetSegNo == readSegNo); - Assert(targetPageOff == readOff); + Assert(targetSegNo == state->seg.ws_segno); Assert(reqLen <= readLen); - xlogreader->seg.ws_tli = curFileTLI; + state->seg.ws_tli = curFileTLI; /* * Check the page header immediately, so that we can retry immediately if @@ -12447,23 +12440,23 @@ retry: * responsible for the validation. */ if (StandbyMode && - !XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + !XLogReaderValidatePageHeader(state, targetPagePtr, readBuf)) { /* * Emit this error right now then retry this page immediately. Use * errmsg_internal() because the message was already translated. */ - if (xlogreader->errormsg_buf[0]) + if (state->errormsg_buf[0]) ereport(emode_for_corrupt_record(emode, EndRecPtr), - (errmsg_internal("%s", xlogreader->errormsg_buf))); + (errmsg_internal("%s", state->errormsg_buf))); /* reset any error XLogReaderValidatePageHeader() might have set */ - xlogreader->errormsg_buf[0] = '\0'; + state->errormsg_buf[0] = '\0'; goto next_record_is_invalid; } - Assert(xlogreader->readPagePtr == targetPagePtr); - xlogreader->readLen = readLen; + Assert(state->readPagePtr == targetPagePtr); + state->readLen = readLen; return true; next_record_is_invalid: @@ -12472,14 +12465,13 @@ next_record_is_invalid: if (readFile >= 0) close(readFile); readFile = -1; - readLen = 0; readSource = XLOG_FROM_ANY; /* In standby-mode, keep trying */ if (StandbyMode) goto retry; - xlogreader->readLen = -1; + state->readLen = -1; return false; } @@ -12511,7 +12503,8 @@ next_record_is_invalid: */ static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess, - bool fetching_ckpt, XLogRecPtr tliRecPtr) + bool fetching_ckpt, XLogRecPtr tliRecPtr, + XLogSegNo readSegNo) { static TimestampTz last_fail_time = 0; TimestampTz now; -- 2.27.0 ----Next_Part(Thu_Oct__7_17_28_20_2021_531)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v18-0004-Make-XLogFindNextRecord-not-use-callback-functio.patch" ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: why there is not VACUUM FULL CONCURRENTLY? @ 2025-02-19 17:08 Antonin Houska <[email protected]> 0 siblings, 1 reply; 58+ messages in thread From: Antonin Houska @ 2025-02-19 17:08 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: Michael Banck <[email protected]>; Junwang Zhao <[email protected]>; Kirill Reshke <[email protected]>; Pavel Stehule <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers Alvaro Herrera <[email protected]> wrote: > On 2025-Jan-30, Michael Banck wrote: > > > > I haven't addressed the problem of a new command yet - for that I'd like to > > > see some sort of consensus, so that I do not have to do all the related > > > changes many times. > > > > Well, looks like this patch-set is blocked on the bikeshedding part? > > > > Somebody should call a shot here, then. > > A bunch of people discussed this patch in today's developer meeting in > Brussels. There's pretty much a consensus on using the verb REPACK > CONCURRENTLY for this new command -- where unadorned REPACK would be > VACUUM FULL, and we'd have something like REPACK WITH INDEX or maybe > REPACK USING INDEX to take the CLUSTER place. This is a patch that adds the REPACK command (w/o CONCURRENTLY). I'll incorporate it into the patch series but it'd be great if this part was a little bit stable before I start to rebase the depending patches. Thanks. -- Antonin Houska Web: https://www.cybertec-postgresql.com -- *E-Mail Disclaimer* Der Inhalt dieser E-Mail ist ausschliesslich fuer den bezeichneten Adressaten bestimmt. Wenn Sie nicht der vorgesehene Adressat dieser E-Mail oder dessen Vertreter sein sollten, so beachten Sie bitte, dass jede Form der Kenntnisnahme, Veroeffentlichung, Vervielfaeltigung oder Weitergabe des Inhalts dieser E-Mail unzulaessig ist. Wir bitten Sie, sich in diesem Fall mit dem Absender der E-Mail in Verbindung zu setzen. *CONFIDENTIALITY NOTICE & DISCLAIMER *This message and any attachment are confidential and may be privileged or otherwise protected from disclosure and solely for the use of the person(s) or entity to whom it is intended. If you have received this message in error and are not the intended recipient, please notify the sender immediately and delete this message and any attachment from your system. If you are not the intended recipient, be advised that any use of this message is prohibited and may be unlawful, and you must not copy this message or attachment or disclose the contents to any other person. Attachments: [text/x-diff] 0001-Add-REPACK-command.patch (89.7K, ../../63871.1739984882@localhost/2-0001-Add-REPACK-command.patch) download | inline diff: From b09ae021ea8fa7b4a90775c25e3f0ddaa711ef82 Mon Sep 17 00:00:00 2001 From: Antonin Houska <[email protected]> Date: Wed, 19 Feb 2025 16:17:37 +0100 Subject: [PATCH] Add REPACK command. The existing CLUSTER command as well as VACUUM with the FULL option both reclaim unused space by rewriting table. Now that we want to enhance this functionality (in particular, by adding a new option CONCURRENTLY), we should enhance both commands because they are both implemented by the same function (cluster.c:cluster_rel). However, adding the same option to two different commands is not very user-friendly. Therefore it was decided to create a new command and to declare both CLUSTER command and the FULL option of VACUUM deprecated. Future enhancements to this rewriting code will only affect the new command. Like CLUSTER, the REPACK command reorders the table according to the specified index. Unlike CLUSTER, REPACK does not require the index: if only table is specified, the command acts as VACUUM FULL. As we don't want to remove CLUSTER and VACUUM FULL yet, there are three callers of the cluster_rel() function now: REPACK, CLUSTER and VACUUM FULL. When we need to distinguish who is calling this function (mostly for logging, but also for progress reporting), we can no longer use the OID of the clustering index: both REPACK and VACUUM FULL can pass InvalidOid. Therefore, this patch introduces a new enumeration type ClusterCommand, and adds an argument of this type to the cluster_rel() function and to all the functions that need to distinguish the caller. Like CLUSTER and VACUUM FULL, the REPACK COMMAND without arguments processes all the tables on which the current user has the MAINTAIN privilege. A new view pg_stat_progress_repack view is added to monitor the progress of REPACK. Currently it displays the same information as pg_stat_progress_cluster (except that column names might differ), but it'll also display the status of the REPACK CONCURRENTLY command in the future, so the view definitions will eventually diverge. Regarding user documentation, the patch moves the information on clustering from cluster.sgml to the new file repack.sgml. cluster.sgml now contains a link that points to the related section of repack.sgml. A note on deprecation and a link to repack.sgml are added to both cluster.sgml and vacuum.sgml. --- doc/src/sgml/monitoring.sgml | 230 +++++++++++ doc/src/sgml/ref/allfiles.sgml | 1 + doc/src/sgml/ref/cluster.sgml | 79 +--- doc/src/sgml/ref/repack.sgml | 254 ++++++++++++ doc/src/sgml/ref/vacuum.sgml | 8 + doc/src/sgml/reference.sgml | 1 + src/backend/access/heap/heapam_handler.c | 32 +- src/backend/catalog/index.c | 2 +- src/backend/catalog/system_views.sql | 27 ++ src/backend/commands/cluster.c | 496 +++++++++++++++++------ src/backend/commands/tablecmds.c | 3 +- src/backend/commands/vacuum.c | 3 +- src/backend/parser/gram.y | 64 ++- src/backend/tcop/utility.c | 9 + src/backend/utils/adt/pgstatfuncs.c | 2 + src/bin/psql/tab-complete.in.c | 31 +- src/include/commands/cluster.h | 22 +- src/include/commands/progress.h | 59 ++- src/include/nodes/parsenodes.h | 13 + src/include/parser/kwlist.h | 1 + src/include/tcop/cmdtaglist.h | 1 + src/include/utils/backend_progress.h | 1 + src/test/regress/expected/cluster.out | 180 ++++++++ src/test/regress/expected/rules.out | 27 ++ src/test/regress/sql/cluster.sql | 73 ++++ src/tools/pgindent/typedefs.list | 2 + 26 files changed, 1385 insertions(+), 236 deletions(-) create mode 100644 doc/src/sgml/ref/repack.sgml diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 928a6eb64b..8a1ed9b645 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -400,6 +400,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_stat_progress_repack</structname><indexterm><primary>pg_stat_progress_repack</primary></indexterm></entry> + <entry>One row for each backend running + <command>REPACK</command>, showing current progress. See + <xref linkend="repack-progress-reporting"/>. + </entry> + </row> + <row> <entry><structname>pg_stat_progress_basebackup</structname><indexterm><primary>pg_stat_progress_basebackup</primary></indexterm></entry> <entry>One row for each WAL sender process streaming a base backup, @@ -5916,6 +5924,228 @@ FROM pg_stat_get_backend_idset() AS backendid; </table> </sect2> + <sect2 id="repack-progress-reporting"> + <title>REPACK Progress Reporting</title> + + <indexterm> + <primary>pg_stat_progress_repack</primary> + </indexterm> + + <para> + Whenever <command>REPACK</command> is running, + the <structname>pg_stat_progress_repack</structname> view will contain a + row for each backend that is currently running the command. The tables + below describe the information that will be reported and provide + information about how to interpret it. + </para> + + <table id="pg-stat-progress-repack-view" xreflabel="pg_stat_progress_repack"> + <title><structname>pg_stat_progress_repack</structname> View</title> + <tgroup cols="1"> + <thead> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + Column Type + </para> + <para> + Description + </para></entry> + </row> + </thead> + + <tbody> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>pid</structfield> <type>integer</type> + </para> + <para> + Process ID of backend. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>datid</structfield> <type>oid</type> + </para> + <para> + OID of the database to which this backend is connected. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>datname</structfield> <type>name</type> + </para> + <para> + Name of the database to which this backend is connected. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>relid</structfield> <type>oid</type> + </para> + <para> + OID of the table being repacked. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>command</structfield> <type>text</type> + </para> + <para> + The command that is running. Currently, the only value + is <literal>REPACK</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>phase</structfield> <type>text</type> + </para> + <para> + Current processing phase. See <xref linkend="repack-phases"/>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>repack_index_relid</structfield> <type>oid</type> + </para> + <para> + If the table is being scanned using an index, this is the OID of the + index being used; otherwise, it is zero. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_tuples_scanned</structfield> <type>bigint</type> + </para> + <para> + Number of heap tuples scanned. + This counter only advances when the phase is + <literal>seq scanning heap</literal>, + <literal>index scanning heap</literal> + or <literal>writing new heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_tuples_written</structfield> <type>bigint</type> + </para> + <para> + Number of heap tuples written. + This counter only advances when the phase is + <literal>seq scanning heap</literal>, + <literal>index scanning heap</literal> + or <literal>writing new heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_blks_total</structfield> <type>bigint</type> + </para> + <para> + Total number of heap blocks in the table. This number is reported + as of the beginning of <literal>seq scanning heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_blks_scanned</structfield> <type>bigint</type> + </para> + <para> + Number of heap blocks scanned. This counter only advances when the + phase is <literal>seq scanning heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>index_rebuild_count</structfield> <type>bigint</type> + </para> + <para> + Number of indexes rebuilt. This counter only advances when the phase + is <literal>rebuilding index</literal>. + </para></entry> + </row> + </tbody> + </tgroup> + </table> + + <table id="repack-phases"> + <title>REPACK Phases</title> + <tgroup cols="2"> + <colspec colname="col1" colwidth="1*"/> + <colspec colname="col2" colwidth="2*"/> + <thead> + <row> + <entry>Phase</entry> + <entry>Description</entry> + </row> + </thead> + + <tbody> + <row> + <entry><literal>initializing</literal></entry> + <entry> + The command is preparing to begin scanning the heap. This phase is + expected to be very brief. + </entry> + </row> + <row> + <entry><literal>seq scanning heap</literal></entry> + <entry> + The command is currently scanning the table using a sequential scan. + </entry> + </row> + <row> + <entry><literal>index scanning heap</literal></entry> + <entry> + <command>REPACK</command> is currently scanning the table using an index scan. + </entry> + </row> + <row> + <entry><literal>sorting tuples</literal></entry> + <entry> + <command>REPACK</command> is currently sorting tuples. + </entry> + </row> + <row> + <entry><literal>writing new heap</literal></entry> + <entry> + <command>REPACK</command> is currently writing the new heap. + </entry> + </row> + <row> + <entry><literal>swapping relation files</literal></entry> + <entry> + The command is currently swapping newly-built files into place. + </entry> + </row> + <row> + <entry><literal>rebuilding index</literal></entry> + <entry> + The command is currently rebuilding an index. + </entry> + </row> + <row> + <entry><literal>performing final cleanup</literal></entry> + <entry> + The command is performing final cleanup. When this phase is + completed, <command>REPACK</command> will end. + </entry> + </row> + </tbody> + </tgroup> + </table> + </sect2> + <sect2 id="copy-progress-reporting"> <title>COPY Progress Reporting</title> diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index f5be638867..c0ef654fcb 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -167,6 +167,7 @@ Complete list of usable sgml source files in this directory. <!ENTITY refreshMaterializedView SYSTEM "refresh_materialized_view.sgml"> <!ENTITY reindex SYSTEM "reindex.sgml"> <!ENTITY releaseSavepoint SYSTEM "release_savepoint.sgml"> +<!ENTITY repack SYSTEM "repack.sgml"> <!ENTITY reset SYSTEM "reset.sgml"> <!ENTITY revoke SYSTEM "revoke.sgml"> <!ENTITY rollback SYSTEM "rollback.sgml"> diff --git a/doc/src/sgml/ref/cluster.sgml b/doc/src/sgml/ref/cluster.sgml index 8811f169ea..54bb2362c8 100644 --- a/doc/src/sgml/ref/cluster.sgml +++ b/doc/src/sgml/ref/cluster.sgml @@ -42,17 +42,23 @@ CLUSTER [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <r <replaceable class="parameter">table_name</replaceable>. </para> - <para> - When a table is clustered, it is physically reordered - based on the index information. Clustering is a one-time operation: - when the table is subsequently updated, the changes are - not clustered. That is, no attempt is made to store new or - updated rows according to their index order. (If one wishes, one can - periodically recluster by issuing the command again. Also, setting - the table's <literal>fillfactor</literal> storage parameter to less than - 100% can aid in preserving cluster ordering during updates, since updated - rows are kept on the same page if enough space is available there.) - </para> + <warning> + <para> + The <command>CLUSTER</command> command is deprecated in favor of + <xref linkend="sql-repack"/>. + </para> + </warning> + + <note> + <para> + <xref linkend="sql-repack-notes-on-clustering"/> explain how clustering + works, whether it is initiated by <command>CLUSTER</command> or + by <command>REPACK</command>. The notable difference between the two is + that <command>REPACK</command> does not remember the index used last + time. Thus if you don't specify an index, <command>REPACK</command> + rewrites the table but does not try to cluster it. + </para> + </note> <para> When a table is clustered, <productname>PostgreSQL</productname> @@ -136,63 +142,12 @@ CLUSTER [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <r on the table. </para> - <para> - In cases where you are accessing single rows randomly - within a table, the actual order of the data in the - table is unimportant. However, if you tend to access some - data more than others, and there is an index that groups - them together, you will benefit from using <command>CLUSTER</command>. - If you are requesting a range of indexed values from a table, or a - single indexed value that has multiple rows that match, - <command>CLUSTER</command> will help because once the index identifies the - table page for the first row that matches, all other rows - that match are probably already on the same table page, - and so you save disk accesses and speed up the query. - </para> - - <para> - <command>CLUSTER</command> can re-sort the table using either an index scan - on the specified index, or (if the index is a b-tree) a sequential - scan followed by sorting. It will attempt to choose the method that - will be faster, based on planner cost parameters and available statistical - information. - </para> - <para> While <command>CLUSTER</command> is running, the <xref linkend="guc-search-path"/> is temporarily changed to <literal>pg_catalog, pg_temp</literal>. </para> - <para> - When an index scan is used, a temporary copy of the table is created that - contains the table data in the index order. Temporary copies of each - index on the table are created as well. Therefore, you need free space on - disk at least equal to the sum of the table size and the index sizes. - </para> - - <para> - When a sequential scan and sort is used, a temporary sort file is - also created, so that the peak temporary space requirement is as much - as double the table size, plus the index sizes. This method is often - faster than the index scan method, but if the disk space requirement is - intolerable, you can disable this choice by temporarily setting <xref - linkend="guc-enable-sort"/> to <literal>off</literal>. - </para> - - <para> - It is advisable to set <xref linkend="guc-maintenance-work-mem"/> to - a reasonably large value (but not more than the amount of RAM you can - dedicate to the <command>CLUSTER</command> operation) before clustering. - </para> - - <para> - Because the planner records statistics about the ordering of - tables, it is advisable to run <link linkend="sql-analyze"><command>ANALYZE</command></link> - on the newly clustered table. - Otherwise, the planner might make poor choices of query plans. - </para> - <para> Because <command>CLUSTER</command> remembers which indexes are clustered, one can cluster the tables one wants clustered manually the first time, diff --git a/doc/src/sgml/ref/repack.sgml b/doc/src/sgml/ref/repack.sgml new file mode 100644 index 0000000000..e2b96c12fb --- /dev/null +++ b/doc/src/sgml/ref/repack.sgml @@ -0,0 +1,254 @@ +<!-- +doc/src/sgml/ref/repack.sgml +PostgreSQL documentation +--> + +<refentry id="sql-repack"> + <indexterm zone="sql-repack"> + <primary>REPACK</primary> + </indexterm> + + <refmeta> + <refentrytitle>REPACK</refentrytitle> + <manvolnum>7</manvolnum> + <refmiscinfo>SQL - Language Statements</refmiscinfo> + </refmeta> + + <refnamediv> + <refname>REPACK</refname> + <refpurpose>cluster a table according to an index</refpurpose> + </refnamediv> + + <refsynopsisdiv> +<synopsis> +REPACK [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <replaceable class="parameter">table_name</replaceable> [ USING INDEX<replaceable class="parameter">index_name</replaceable> ] ] + +<phrase>where <replaceable class="parameter">option</replaceable> can be one of:</phrase> + + VERBOSE [ <replaceable class="parameter">boolean</replaceable> ] +</synopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>REPACK</command> reclaims storage occupied by dead + tuples. Unlike <command>VACUUM</command>, it does so by rewriting the + entire contents of the table specified + by <replaceable class="parameter">table_name</replaceable> into a new disk + file with no extra space, allowing unused space to be returned to the + operating + system. + </para> + + <para> + Without + a <replaceable class="parameter">table_name</replaceable>, <command>REPACK</command> + processes every table and materialized view in the current database that + the current user has the <literal>MAINTAIN</literal> privilege on. This + form of <command>REPACK</command> cannot be executed inside a transaction + block. + </para> + + <para> + If <replaceable class="parameter">index_name</replaceable> is specified, + the table is clustered by this index. Please see the notes on clustering + below. + </para> + + <para> + When a table is being repacked, an <literal>ACCESS EXCLUSIVE</literal> lock + is acquired on it. This prevents any other database operations (both reads + and writes) from operating on the table until the <command>REPACK</command> + is finished. + </para> + + <refsect2 id="sql-repack-notes-on-clustering" xreflabel="Notes on Clustering"> + <title>Notes on Clustering</title> + + <para> + When a table is clustered, it is physically reordered based on the index + information. Clustering is a one-time operation: when the table is + subsequently updated, the changes are not clustered. That is, no attempt + is made to store new or updated rows according to their index order. (If + one wishes, one can periodically recluster by issuing the command again. + Also, setting the table's <literal>fillfactor</literal> storage parameter + to less than 100% can aid in preserving cluster ordering during updates, + since updated rows are kept on the same page if enough space is available + there.) + </para> + + <para> + In cases where you are accessing single rows randomly within a table, the + actual order of the data in the table is unimportant. However, if you tend + to access some data more than others, and there is an index that groups + them together, you will benefit from using <command>REPACK</command>. If + you are requesting a range of indexed values from a table, or a single + indexed value that has multiple rows that match, + <command>REPACK</command> will help because once the index identifies the + table page for the first row that matches, all other rows that match are + probably already on the same table page, and so you save disk accesses and + speed up the query. + </para> + + <para> + <command>REPACK</command> can re-sort the table using either an index scan + on the specified index (if the index is a b-tree), or a sequential scan + followed by sorting. It will attempt to choose the method that will be + faster, based on planner cost parameters and available statistical + information. + </para> + + <para> + Because the planner records statistics about the ordering of tables, it is + advisable to + run <link linkend="sql-analyze"><command>ANALYZE</command></link> on the + newly repacked table. Otherwise, the planner might make poor choices of + query plans. + </para> + </refsect2> + + <refsect2 id="sql-repack-notes-on-resources" xreflabel="Notes on Resources"> + <title>Notes on Resources</title> + + <para> + When an index scan or a sequential scan without sort is used, a temporary + copy of the table is created that contains the table data in the index + order. Temporary copies of each index on the table are created as well. + Therefore, you need free space on disk at least equal to the sum of the + table size and the index sizes. + </para> + + <para> + When a sequential scan and sort is used, a temporary sort file is also + created, so that the peak temporary space requirement is as much as double + the table size, plus the index sizes. This method is often faster than + the index scan method, but if the disk space requirement is intolerable, + you can disable this choice by temporarily setting + <xref linkend="guc-enable-sort"/> to <literal>off</literal>. + </para> + + <para> + It is advisable to set <xref linkend="guc-maintenance-work-mem"/> to a + reasonably large value (but not more than the amount of RAM you can + dedicate to the <command>REPACK</command> operation) before repacking. + </para> + </refsect2> + + </refsect1> + + <refsect1> + <title>Parameters</title> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">table_name</replaceable></term> + <listitem> + <para> + The name (possibly schema-qualified) of a table. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><replaceable class="parameter">index_name</replaceable></term> + <listitem> + <para> + The name of an index. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><literal>VERBOSE</literal></term> + <listitem> + <para> + Prints a progress report as each table is clustered + at <literal>INFO</literal> level. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><replaceable class="parameter">boolean</replaceable></term> + <listitem> + <para> + Specifies whether the selected option should be turned on or off. + You can write <literal>TRUE</literal>, <literal>ON</literal>, or + <literal>1</literal> to enable the option, and <literal>FALSE</literal>, + <literal>OFF</literal>, or <literal>0</literal> to disable it. The + <replaceable class="parameter">boolean</replaceable> value can also + be omitted, in which case <literal>TRUE</literal> is assumed. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1> + <title>Notes</title> + + <para> + To repack a table, one must have the <literal>MAINTAIN</literal> privilege + on the table. + </para> + + <para> + While <command>REPACK</command> is running, the <xref + linkend="guc-search-path"/> is temporarily changed to <literal>pg_catalog, + pg_temp</literal>. + </para> + + <para> + Each backend running <command>REPACK</command> will report its progress + in the <structname>pg_stat_progress_repack</structname> view. See + <xref linkend="repack-progress-reporting"/> for details. + </para> + + <para> + Repacking a partitioned table repacks each of its partitions. If an index + is specified, each partition is clustered using the partition of that + index. <command>REPACK</command> on a partitioned table cannot be executed + inside a transaction block. + </para> + + </refsect1> + + <refsect1> + <title>Examples</title> + + <para> + Repack the table <literal>employees</literal>: +<programlisting> +REPACK employees; +</programlisting> + </para> + + + <para> + Cluster the table <literal>employees</literal> on the basis of its + index <literal>employees_ind</literal>: +<programlisting> +REPACK employees USING INDEX employees_ind; +</programlisting> + </para> + + <para> + Repack all tables in the database on which you have + the <literal>MAINTAIN</literal> privilege: +<programlisting> +REPACK; +</programlisting></para> + </refsect1> + + <refsect1> + <title>Compatibility</title> + + <para> + There is no <command>REPACK</command> statement in the SQL standard. + </para> + + </refsect1> + +</refentry> diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 971b1237d4..2b5a5d0ac4 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -98,6 +98,14 @@ VACUUM [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <re <varlistentry> <term><literal>FULL</literal></term> <listitem> + + <warning> + <para> + The <command>FULL</command> parameter is deprecated in favor of + <xref linkend="sql-repack"/>. + </para> + </warning> + <para> Selects <quote>full</quote> vacuum, which can reclaim more space, but takes much longer and exclusively locks the table. diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index ff85ace83f..229912d35b 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -195,6 +195,7 @@ &refreshMaterializedView; &reindex; &releaseSavepoint; + &repack; &reset; &revoke; &rollback; diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index c0bec01415..5c9dcc938d 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -737,13 +737,13 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, if (OldIndex != NULL && !use_sort) { const int ci_index[] = { - PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_INDEX_RELID + PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_INDEX_RELID }; int64 ci_val[2]; /* Set phase and OIDOldIndex to columns */ - ci_val[0] = PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP; + ci_val[0] = PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP; ci_val[1] = RelationGetRelid(OldIndex); pgstat_progress_update_multi_param(2, ci_index, ci_val); @@ -755,15 +755,15 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, else { /* In scan-and-sort mode and also VACUUM FULL, set phase */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP); tableScan = table_beginscan(OldHeap, SnapshotAny, 0, (ScanKey) NULL); heapScan = (HeapScanDesc) tableScan; indexScan = NULL; /* Set total heap blocks */ - pgstat_progress_update_param(PROGRESS_CLUSTER_TOTAL_HEAP_BLKS, + pgstat_progress_update_param(PROGRESS_REPACK_TOTAL_HEAP_BLKS, heapScan->rs_nblocks); } @@ -805,7 +805,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, * is manually updated to the correct value when the table * scan finishes. */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_BLKS_SCANNED, heapScan->rs_nblocks); break; } @@ -821,7 +821,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, */ if (prev_cblock != heapScan->rs_cblock) { - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_BLKS_SCANNED, (heapScan->rs_cblock + heapScan->rs_nblocks - heapScan->rs_startblock @@ -908,14 +908,14 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, * In scan-and-sort mode, report increase in number of tuples * scanned */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_SCANNED, *num_tuples); } else { const int ct_index[] = { - PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, - PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN + PROGRESS_REPACK_HEAP_TUPLES_SCANNED, + PROGRESS_REPACK_HEAP_TUPLES_WRITTEN }; int64 ct_val[2]; @@ -948,14 +948,14 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, double n_tuples = 0; /* Report that we are now sorting tuples */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SORT_TUPLES); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SORT_TUPLES); tuplesort_performsort(tuplesort); /* Report that we are now writing new heap */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP); for (;;) { @@ -973,7 +973,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, values, isnull, rwstate); /* Report n_tuples */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_WRITTEN, n_tuples); } diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index cdabf78024..a3c93eea5f 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -4052,7 +4052,7 @@ reindex_relation(const ReindexStmt *stmt, Oid relid, int flags, Assert(!ReindexIsProcessingIndex(indexOid)); /* Set index rebuild count */ - pgstat_progress_update_param(PROGRESS_CLUSTER_INDEX_REBUILD_COUNT, + pgstat_progress_update_param(PROGRESS_REPACK_INDEX_REBUILD_COUNT, i); i++; } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index eff0990957..412627e22b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1266,6 +1266,33 @@ CREATE VIEW pg_stat_progress_cluster AS FROM pg_stat_get_progress_info('CLUSTER') AS S LEFT JOIN pg_database D ON S.datid = D.oid; +CREATE VIEW pg_stat_progress_repack AS + SELECT + S.pid AS pid, + S.datid AS datid, + D.datname AS datname, + S.relid AS relid, + CASE S.param1 WHEN 1 THEN 'REPACK' + END AS command, + CASE S.param2 WHEN 0 THEN 'initializing' + WHEN 1 THEN 'seq scanning heap' + WHEN 2 THEN 'index scanning heap' + WHEN 3 THEN 'sorting tuples' + WHEN 4 THEN 'writing new heap' + WHEN 5 THEN 'swapping relation files' + WHEN 6 THEN 'rebuilding index' + WHEN 7 THEN 'performing final cleanup' + END AS phase, + CAST(S.param3 AS oid) AS repack_index_relid, + S.param4 AS heap_tuples_scanned, + S.param5 AS heap_tuples_written, + S.param6 AS heap_blks_total, + S.param7 AS heap_blks_scanned, + S.param8 AS index_rebuild_count + FROM pg_stat_get_progress_info('REPACK') AS S + LEFT JOIN pg_database D ON S.datid = D.oid; + + CREATE VIEW pg_stat_progress_create_index AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 99193f5c88..d0f2588a97 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -46,6 +46,7 @@ #include "storage/lmgr.h" #include "storage/predicate.h" #include "utils/acl.h" +#include "utils/formatting.h" #include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/inval.h" @@ -67,17 +68,33 @@ typedef struct Oid indexOid; } RelToCluster; - -static void cluster_multiple_rels(List *rtcs, ClusterParams *params); -static void rebuild_relation(Relation OldHeap, Relation index, bool verbose); +/* + * Map the value of ClusterCommand to string. + */ +#define CLUSTER_COMMAND_STR(cmd) ((cmd) == CLUSTER_COMMAND_CLUSTER ? \ + "cluster" : \ + ((cmd) == CLUSTER_COMMAND_REPACK ? \ + "repack" : "vacuum")) + +static void cluster_multiple_rels(List *rtcs, ClusterParams *params, + ClusterCommand cmd); +static void rebuild_relation(Relation OldHeap, Relation index, bool verbose, + ClusterCommand cmd); static void copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, - bool verbose, bool *pSwapToastByContent, + bool verbose, ClusterCommand cmd, + bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti); static List *get_tables_to_cluster(MemoryContext cluster_context); +static List *get_tables_to_repack(MemoryContext repack_context); static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context, - Oid indexOid); -static bool cluster_is_permitted_for_relation(Oid relid, Oid userid); - + Oid relid, bool rel_is_index, + ClusterCommand cmd); +static bool cluster_is_permitted_for_relation(Oid relid, Oid userid, + ClusterCommand cmd); +static Relation process_single_relation(RangeVar *relation, char *indexname, + ClusterCommand cmd, + ClusterParams *params, + Oid *indexOid_p); /*--------------------------------------------------------------------------- * This cluster code allows for clustering multiple tables at once. Because @@ -133,72 +150,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) if (stmt->relation != NULL) { - /* This is the single-relation case. */ - Oid tableOid; - - /* - * Find, lock, and check permissions on the table. We obtain - * AccessExclusiveLock right away to avoid lock-upgrade hazard in the - * single-transaction case. - */ - tableOid = RangeVarGetRelidExtended(stmt->relation, - AccessExclusiveLock, - 0, - RangeVarCallbackMaintainsTable, - NULL); - rel = table_open(tableOid, NoLock); - - /* - * Reject clustering a remote temp table ... their local buffer - * manager is not going to cope. - */ - if (RELATION_IS_OTHER_TEMP(rel)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster temporary tables of other sessions"))); - - if (stmt->indexname == NULL) - { - ListCell *index; - - /* We need to find the index that has indisclustered set. */ - foreach(index, RelationGetIndexList(rel)) - { - indexOid = lfirst_oid(index); - if (get_index_isclustered(indexOid)) - break; - indexOid = InvalidOid; - } - - if (!OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("there is no previously clustered index for table \"%s\"", - stmt->relation->relname))); - } - else - { - /* - * The index is expected to be in the same namespace as the - * relation. - */ - indexOid = get_relname_relid(stmt->indexname, - rel->rd_rel->relnamespace); - if (!OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("index \"%s\" for table \"%s\" does not exist", - stmt->indexname, stmt->relation->relname))); - } - - /* For non-partitioned tables, do what we came here to do. */ - if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) - { - cluster_rel(rel, indexOid, ¶ms); - /* cluster_rel closes the relation, but keeps lock */ - + rel = process_single_relation(stmt->relation, stmt->indexname, + CLUSTER_COMMAND_CLUSTER, ¶ms, + &indexOid); + if (rel == NULL) return; - } } /* @@ -230,8 +186,11 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) if (rel != NULL) { Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); - check_index_is_clusterable(rel, indexOid, AccessShareLock); - rtcs = get_tables_to_cluster_partitioned(cluster_context, indexOid); + check_index_is_clusterable(rel, indexOid, AccessShareLock, + CLUSTER_COMMAND_CLUSTER); + rtcs = get_tables_to_cluster_partitioned(cluster_context, indexOid, + true, + CLUSTER_COMMAND_CLUSTER); /* close relation, releasing lock on parent table */ table_close(rel, AccessExclusiveLock); @@ -243,7 +202,7 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) } /* Do the job. */ - cluster_multiple_rels(rtcs, ¶ms); + cluster_multiple_rels(rtcs, ¶ms, CLUSTER_COMMAND_CLUSTER); /* Start a new transaction for the cleanup work. */ StartTransactionCommand(); @@ -260,7 +219,8 @@ cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) * return. */ static void -cluster_multiple_rels(List *rtcs, ClusterParams *params) +cluster_multiple_rels(List *rtcs, ClusterParams *params, + ClusterCommand cmd) { ListCell *lc; @@ -283,7 +243,7 @@ cluster_multiple_rels(List *rtcs, ClusterParams *params) rel = table_open(rtc->tableOid, AccessExclusiveLock); /* Process this table */ - cluster_rel(rel, rtc->indexOid, params); + cluster_rel(rel, rtc->indexOid, params, cmd); /* cluster_rel closes the relation, but keeps lock */ PopActiveSnapshot(); @@ -306,9 +266,13 @@ cluster_multiple_rels(List *rtcs, ClusterParams *params) * If indexOid is InvalidOid, the table will be rewritten in physical order * instead of index order. This is the new implementation of VACUUM FULL, * and error messages should refer to the operation as VACUUM not CLUSTER. + * + * 'cmd' indicates which commands is being executed. REPACK should be the only + * caller of this function in the future. */ void -cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) +cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params, + ClusterCommand cmd) { Oid tableOid = RelationGetRelid(OldHeap); Oid save_userid; @@ -317,19 +281,33 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) bool verbose = ((params->options & CLUOPT_VERBOSE) != 0); bool recheck = ((params->options & CLUOPT_RECHECK) != 0); Relation index; + const char *cmd_str = CLUSTER_COMMAND_STR(cmd); Assert(CheckRelationLockedByMe(OldHeap, AccessExclusiveLock, false)); /* Check for user-requested abort. */ CHECK_FOR_INTERRUPTS(); - pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid); - if (OidIsValid(indexOid)) - pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, + if (cmd == CLUSTER_COMMAND_REPACK) + pgstat_progress_start_command(PROGRESS_COMMAND_REPACK, tableOid); + else + pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid); + + if (cmd == CLUSTER_COMMAND_REPACK) + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, + PROGRESS_REPACK_COMMAND_REPACK); + else if (OidIsValid(indexOid)) + { + Assert(cmd == CLUSTER_COMMAND_CLUSTER); + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, PROGRESS_CLUSTER_COMMAND_CLUSTER); + } else - pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, + { + Assert(cmd == CLUSTER_COMMAND_VACUUM); + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, PROGRESS_CLUSTER_COMMAND_VACUUM_FULL); + } /* * Switch to the table owner's userid, so that any index functions are run @@ -353,7 +331,7 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) if (recheck) { /* Check that the user still has privileges for the relation */ - if (!cluster_is_permitted_for_relation(tableOid, save_userid)) + if (!cluster_is_permitted_for_relation(tableOid, save_userid, cmd)) { relation_close(OldHeap, AccessExclusiveLock); goto out; @@ -403,39 +381,38 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) * would work in most respects, but the index would only get marked as * indisclustered in the current database, leading to unexpected behavior * if CLUSTER were later invoked in another database. + * + * REPACK does not set indisclustered. XXX Not sure I understand the + * comment above: how can an attribute be set "only in the current + * database"? */ - if (OidIsValid(indexOid) && OldHeap->rd_rel->relisshared) + if (cmd == CLUSTER_COMMAND_CLUSTER && OldHeap->rd_rel->relisshared) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster a shared catalog"))); + errmsg("cannot %s a shared catalog", cmd_str))); /* * Don't process temp tables of other backends ... their local buffer * manager is not going to cope. */ if (RELATION_IS_OTHER_TEMP(OldHeap)) - { - if (OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster temporary tables of other sessions"))); - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot vacuum temporary tables of other sessions"))); - } + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot %s temporary tables of other sessions", + cmd_str))); /* * Also check for active uses of the relation in the current transaction, * including open scans and pending AFTER trigger events. */ - CheckTableNotInUse(OldHeap, OidIsValid(indexOid) ? "CLUSTER" : "VACUUM"); + CheckTableNotInUse(OldHeap, asc_toupper(cmd_str, strlen(cmd_str))); /* Check heap and index are valid to cluster on */ if (OidIsValid(indexOid)) { /* verify the index is good and lock it */ - check_index_is_clusterable(OldHeap, indexOid, AccessExclusiveLock); + check_index_is_clusterable(OldHeap, indexOid, AccessExclusiveLock, + cmd); /* also open it */ index = index_open(indexOid, NoLock); } @@ -469,7 +446,7 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) TransferPredicateLocksToHeapRelation(OldHeap); /* rebuild_relation does all the dirty work */ - rebuild_relation(OldHeap, index, verbose); + rebuild_relation(OldHeap, index, verbose, cmd); /* rebuild_relation closes OldHeap, and index if valid */ out: @@ -491,9 +468,11 @@ out: * protection here. */ void -check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode) +check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode, + ClusterCommand cmd) { Relation OldIndex; + const char *cmd_str = CLUSTER_COMMAND_STR(cmd); OldIndex = index_open(indexOid, lockmode); @@ -512,8 +491,8 @@ check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode) if (!OldIndex->rd_indam->amclusterable) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster on index \"%s\" because access method does not support clustering", - RelationGetRelationName(OldIndex)))); + errmsg("cannot %s on index \"%s\" because access method does not support clustering", + cmd_str, RelationGetRelationName(OldIndex)))); /* * Disallow clustering on incomplete indexes (those that might not index @@ -524,7 +503,8 @@ check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode) if (!heap_attisnull(OldIndex->rd_indextuple, Anum_pg_index_indpred, NULL)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster on partial index \"%s\"", + errmsg("cannot %s on partial index \"%s\"", + cmd_str, RelationGetRelationName(OldIndex)))); /* @@ -538,8 +518,8 @@ check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode) if (!OldIndex->rd_index->indisvalid) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster on invalid index \"%s\"", - RelationGetRelationName(OldIndex)))); + errmsg("cannot %s on invalid index \"%s\"", + cmd_str, RelationGetRelationName(OldIndex)))); /* Drop relcache refcnt on OldIndex, but keep lock */ index_close(OldIndex, NoLock); @@ -626,7 +606,8 @@ mark_index_clustered(Relation rel, Oid indexOid, bool is_internal) * On exit, they are closed, but locks on them are not released. */ static void -rebuild_relation(Relation OldHeap, Relation index, bool verbose) +rebuild_relation(Relation OldHeap, Relation index, bool verbose, + ClusterCommand cmd) { Oid tableOid = RelationGetRelid(OldHeap); Oid accessMethod = OldHeap->rd_rel->relam; @@ -664,7 +645,7 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose) NewHeap = table_open(OIDNewHeap, NoLock); /* Copy the heap data into the new table in the desired order */ - copy_table_data(NewHeap, OldHeap, index, verbose, + copy_table_data(NewHeap, OldHeap, index, verbose, cmd, &swap_toast_by_content, &frozenXid, &cutoffMulti); @@ -829,8 +810,8 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, Oid NewAccessMethod, */ static void copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, bool verbose, - bool *pSwapToastByContent, TransactionId *pFreezeXid, - MultiXactId *pCutoffMulti) + ClusterCommand cmd, bool *pSwapToastByContent, + TransactionId *pFreezeXid, MultiXactId *pCutoffMulti) { Relation relRelation; HeapTuple reltup; @@ -845,6 +826,7 @@ copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, bool verb tups_recently_dead = 0; BlockNumber num_pages; int elevel = verbose ? INFO : DEBUG2; + const char *cmd_str = CLUSTER_COMMAND_STR(cmd); PGRUsage ru0; char *nspname; @@ -958,18 +940,21 @@ copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, bool verb /* Log what we're doing */ if (OldIndex != NULL && !use_sort) ereport(elevel, - (errmsg("clustering \"%s.%s\" using index scan on \"%s\"", + (errmsg("%sing \"%s.%s\" using index scan on \"%s\"", + cmd_str, nspname, RelationGetRelationName(OldHeap), RelationGetRelationName(OldIndex)))); else if (use_sort) ereport(elevel, - (errmsg("clustering \"%s.%s\" using sequential scan and sort", + (errmsg("%sing \"%s.%s\" using sequential scan and sort", + cmd_str, nspname, RelationGetRelationName(OldHeap)))); else ereport(elevel, - (errmsg("vacuuming \"%s.%s\"", + (errmsg("%sing \"%s.%s\"", + cmd_str, nspname, RelationGetRelationName(OldHeap)))); @@ -1453,8 +1438,8 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, int i; /* Report that we are now swapping relation files */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SWAP_REL_FILES); /* Zero out possible results from swapped_relation_files */ memset(mapped_tables, 0, sizeof(mapped_tables)); @@ -1504,14 +1489,14 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; /* Report that we are now reindexing relations */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_REBUILD_INDEX); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_REBUILD_INDEX); reindex_relation(NULL, OIDOldHeap, reindex_flags, &reindex_params); /* Report that we are now doing clean up */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_FINAL_CLEANUP); /* * If the relation being rebuilt is pg_class, swap_relation_files() @@ -1661,7 +1646,8 @@ get_tables_to_cluster(MemoryContext cluster_context) index = (Form_pg_index) GETSTRUCT(indexTuple); - if (!cluster_is_permitted_for_relation(index->indrelid, GetUserId())) + if (!cluster_is_permitted_for_relation(index->indrelid, GetUserId(), + CLUSTER_COMMAND_CLUSTER)) continue; /* Use a permanent memory context for the result list */ @@ -1682,14 +1668,67 @@ get_tables_to_cluster(MemoryContext cluster_context) } /* - * Given an index on a partitioned table, return a list of RelToCluster for + * Like get_tables_to_cluster(), but do not care about indexes. + */ +static List * +get_tables_to_repack(MemoryContext repack_context) +{ + Relation relrelation; + TableScanDesc scan; + HeapTuple tuple; + MemoryContext old_context; + List *rtcs = NIL; + + /* + * Get all indexes that have indisclustered set and that the current user + * has the appropriate privileges for. + */ + relrelation = table_open(RelationRelationId, AccessShareLock); + scan = table_beginscan_catalog(relrelation, 0, NULL); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + RelToCluster *rtc; + Form_pg_class relrelation = (Form_pg_class) GETSTRUCT(tuple); + Oid relid = relrelation->oid; + + /* Only interested in relations. */ + if (get_rel_relkind(relid) != RELKIND_RELATION) + continue; + + if (!cluster_is_permitted_for_relation(relid, GetUserId(), + CLUSTER_COMMAND_REPACK)) + continue; + + /* Use a permanent memory context for the result list */ + old_context = MemoryContextSwitchTo(repack_context); + + rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); + rtc->tableOid = relid; + rtc->indexOid = InvalidOid; + rtcs = lappend(rtcs, rtc); + + MemoryContextSwitchTo(old_context); + } + table_endscan(scan); + + relation_close(relrelation, AccessShareLock); + + return rtcs; +} + +/* + * Given a partitioned table or its index, return a list of RelToCluster for * all the children leaves tables/indexes. * * Like expand_vacuum_rel, but here caller must hold AccessExclusiveLock * on the table containing the index. + * + * 'rel_is_index' tells whether 'relid' is that of an index (true) or of the + * owning relation. */ static List * -get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) +get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid relid, + bool rel_is_index, ClusterCommand cmd) { List *inhoids; ListCell *lc; @@ -1697,17 +1736,33 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) MemoryContext old_context; /* Do not lock the children until they're processed */ - inhoids = find_all_inheritors(indexOid, NoLock, NULL); + inhoids = find_all_inheritors(relid, NoLock, NULL); foreach(lc, inhoids) { - Oid indexrelid = lfirst_oid(lc); - Oid relid = IndexGetRelation(indexrelid, false); + Oid inhoid = lfirst_oid(lc); + Oid inhrelid, + inhindid; RelToCluster *rtc; - /* consider only leaf indexes */ - if (get_rel_relkind(indexrelid) != RELKIND_INDEX) - continue; + if (rel_is_index) + { + /* consider only leaf indexes */ + if (get_rel_relkind(inhoid) != RELKIND_INDEX) + continue; + + inhrelid = IndexGetRelation(inhoid, false); + inhindid = inhoid; + } + else + { + /* consider only leaf relations */ + if (get_rel_relkind(inhoid) != RELKIND_RELATION) + continue; + + inhrelid = inhoid; + inhindid = InvalidOid; + } /* * It's possible that the user does not have privileges to CLUSTER the @@ -1715,15 +1770,15 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) * table. We skip any partitions which the user is not permitted to * CLUSTER. */ - if (!cluster_is_permitted_for_relation(relid, GetUserId())) + if (!cluster_is_permitted_for_relation(inhrelid, GetUserId(), cmd)) continue; /* Use a permanent memory context for the result list */ old_context = MemoryContextSwitchTo(cluster_context); rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); - rtc->tableOid = relid; - rtc->indexOid = indexrelid; + rtc->tableOid = inhrelid; + rtc->indexOid = inhindid; rtcs = lappend(rtcs, rtc); MemoryContextSwitchTo(old_context); @@ -1737,13 +1792,192 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) * function emits a WARNING. */ static bool -cluster_is_permitted_for_relation(Oid relid, Oid userid) +cluster_is_permitted_for_relation(Oid relid, Oid userid, ClusterCommand cmd) { if (pg_class_aclcheck(relid, userid, ACL_MAINTAIN) == ACLCHECK_OK) return true; ereport(WARNING, - (errmsg("permission denied to cluster \"%s\", skipping it", + (errmsg("permission denied to %s \"%s\", skipping it", + CLUSTER_COMMAND_STR(cmd), get_rel_name(relid)))); return false; } + +/* + * REPACK is intended to be a replacement of both CLUSTER and VACUUM FULL. + */ +void +repack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel) +{ + ListCell *lc; + ClusterParams params = {0}; + bool verbose = false; + Relation rel = NULL; + Oid indexOid = InvalidOid; + MemoryContext repack_context; + List *rtcs; + + /* Parse option list */ + foreach(lc, stmt->params) + { + DefElem *opt = (DefElem *) lfirst(lc); + + if (strcmp(opt->defname, "verbose") == 0) + verbose = defGetBoolean(opt); + else + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized REPACK option \"%s\"", + opt->defname), + parser_errposition(pstate, opt->location))); + } + + params.options = (verbose ? CLUOPT_VERBOSE : 0); + + if (stmt->relation != NULL) + { + rel = process_single_relation(stmt->relation, stmt->indexname, + CLUSTER_COMMAND_REPACK, ¶ms, + &indexOid); + if (rel == NULL) + return; + } + + /* + * By here, we know we are in a multi-table situation. In order to avoid + * holding locks for too long, we want to process each table in its own + * transaction. This forces us to disallow running inside a user + * transaction block. + */ + PreventInTransactionBlock(isTopLevel, "REPACK"); + + /* Also, we need a memory context to hold our list of relations */ + repack_context = AllocSetContextCreate(PortalContext, + "Repack", + ALLOCSET_DEFAULT_SIZES); + + params.options |= CLUOPT_RECHECK; + if (rel != NULL) + { + Oid relid; + bool rel_is_index; + + Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + + if (OidIsValid(indexOid)) + { + relid = indexOid; + rel_is_index = true; + } + else + { + relid = RelationGetRelid(rel); + rel_is_index = false; + } + rtcs = get_tables_to_cluster_partitioned(repack_context, relid, + rel_is_index, + CLUSTER_COMMAND_REPACK); + + /* close relation, releasing lock on parent table */ + table_close(rel, AccessExclusiveLock); + } + else + rtcs = get_tables_to_repack(repack_context); + + /* Do the job. */ + cluster_multiple_rels(rtcs, ¶ms, CLUSTER_COMMAND_REPACK); + + /* Start a new transaction for the cleanup work. */ + StartTransactionCommand(); + + /* Clean up working storage */ + MemoryContextDelete(repack_context); + +} + +/* + * REPACK a single relation. + * + * Return NULL if done, relation reference if the caller needs to process it + * (because the relation is partitioned). + */ +static Relation +process_single_relation(RangeVar *relation, char *indexname, + ClusterCommand cmd, ClusterParams *params, + Oid *indexOid_p) +{ + Relation rel; + Oid indexOid = InvalidOid; + + /* This is the single-relation case. */ + Oid tableOid; + + /* + * Find, lock, and check permissions on the table. We obtain + * AccessExclusiveLock right away to avoid lock-upgrade hazard in the + * single-transaction case. + */ + tableOid = RangeVarGetRelidExtended(relation, + AccessExclusiveLock, + 0, + RangeVarCallbackMaintainsTable, + NULL); + rel = table_open(tableOid, NoLock); + + /* + * Reject clustering a remote temp table ... their local buffer manager is + * not going to cope. + */ + if (RELATION_IS_OTHER_TEMP(rel)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot %s temporary tables of other sessions", + CLUSTER_COMMAND_STR(cmd)))); + + if (indexname == NULL && cmd == CLUSTER_COMMAND_CLUSTER) + { + ListCell *index; + + /* We need to find the index that has indisclustered set. */ + foreach(index, RelationGetIndexList(rel)) + { + indexOid = lfirst_oid(index); + if (get_index_isclustered(indexOid)) + break; + indexOid = InvalidOid; + } + + if (!OidIsValid(indexOid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("there is no previously clustered index for table \"%s\"", + relation->relname))); + } + else if (indexname != NULL) + { + /* + * The index is expected to be in the same namespace as the relation. + */ + indexOid = get_relname_relid(indexname, + rel->rd_rel->relnamespace); + if (!OidIsValid(indexOid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("index \"%s\" for table \"%s\" does not exist", + indexname, relation->relname))); + } + + *indexOid_p = indexOid; + + /* For non-partitioned tables, do what we came here to do. */ + if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) + { + cluster_rel(rel, indexOid, params, cmd); + /* cluster_rel closes the relation, but keeps lock */ + + return NULL; + } + + return rel; +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 72a1b64c2a..25d21d8838 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -15525,7 +15525,8 @@ ATExecClusterOn(Relation rel, const char *indexName, LOCKMODE lockmode) indexName, RelationGetRelationName(rel)))); /* Check index is valid to cluster on */ - check_index_is_clusterable(rel, indexOid, lockmode); + check_index_is_clusterable(rel, indexOid, lockmode, + CLUSTER_COMMAND_CLUSTER); /* And do the work */ mark_index_clustered(rel, indexOid, false); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 0239d9bae6..59dddcd31f 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -2248,7 +2248,8 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, cluster_params.options |= CLUOPT_VERBOSE; /* VACUUM FULL is now a variant of CLUSTER; see cluster.c */ - cluster_rel(rel, InvalidOid, &cluster_params); + cluster_rel(rel, InvalidOid, &cluster_params, + CLUSTER_COMMAND_VACUUM); /* cluster_rel closes the relation, but keeps lock */ rel = NULL; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index d3887628d4..3077606a17 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -298,7 +298,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); GrantStmt GrantRoleStmt ImportForeignSchemaStmt IndexStmt InsertStmt ListenStmt LoadStmt LockStmt MergeStmt NotifyStmt ExplainableStmt PreparableStmt CreateFunctionStmt AlterFunctionStmt ReindexStmt RemoveAggrStmt - RemoveFuncStmt RemoveOperStmt RenameStmt ReturnStmt RevokeStmt RevokeRoleStmt + RemoveFuncStmt RemoveOperStmt RenameStmt RepackStmt ReturnStmt RevokeStmt RevokeRoleStmt RuleActionStmt RuleActionStmtOrEmpty RuleStmt SecLabelStmt SelectStmt TransactionStmt TransactionStmtLegacy TruncateStmt UnlistenStmt UpdateStmt VacuumStmt @@ -381,7 +381,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <str> copy_file_name access_method_clause attr_name table_access_method_clause name cursor_name file_name - cluster_index_specification + cluster_index_specification repack_index_specification %type <list> func_name handler_name qual_Op qual_all_Op subquery_Op opt_inline_handler opt_validator validator_clause @@ -764,7 +764,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); QUOTE QUOTES RANGE READ REAL REASSIGN RECURSIVE REF_P REFERENCES REFERENCING - REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA + REFRESH REINDEX RELATIVE_P RELEASE RENAME REPACK REPEATABLE REPLACE REPLICA RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROUTINE ROUTINES ROW ROWS RULE @@ -1100,6 +1100,7 @@ stmt: | RemoveFuncStmt | RemoveOperStmt | RenameStmt + | RepackStmt | RevokeStmt | RevokeRoleStmt | RuleStmt @@ -11869,6 +11870,61 @@ cluster_index_specification: | /*EMPTY*/ { $$ = NULL; } ; +/***************************************************************************** + * + * QUERY: + * REPACK <qualified_name> [ USING INDEX <index_name> ] + * REPACK (options) <qualified_name> [ USING INDEX <index_name> ] + * + *****************************************************************************/ + +RepackStmt: + REPACK qualified_name repack_index_specification + { + RepackStmt *n = makeNode(RepackStmt); + + n->relation = $2; + n->indexname = $3; + n->params = NIL; + $$ = (Node *) n; + } + + | REPACK '(' utility_option_list ')' qualified_name repack_index_specification + { + RepackStmt *n = makeNode(RepackStmt); + + n->relation = $5; + n->indexname = $6; + n->params = $3; + $$ = (Node *) n; + } + + | REPACK + { + RepackStmt *n = makeNode(RepackStmt); + + n->relation = NULL; + n->indexname = NULL; + n->params = NIL; + $$ = (Node *) n; + } + + | REPACK '(' utility_option_list ')' + { + RepackStmt *n = makeNode(RepackStmt); + + n->relation = NULL; + n->indexname = NULL; + n->params = $3; + $$ = (Node *) n; + } + ; + +repack_index_specification: + USING INDEX name { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + /***************************************************************************** * @@ -17909,6 +17965,7 @@ unreserved_keyword: | RELATIVE_P | RELEASE | RENAME + | REPACK | REPEATABLE | REPLACE | REPLICA @@ -18540,6 +18597,7 @@ bare_label_keyword: | RELATIVE_P | RELEASE | RENAME + | REPACK | REPEATABLE | REPLACE | REPLICA diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 25fe3d5801..bf3ba3c2ae 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -280,6 +280,7 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_ClusterStmt: case T_ReindexStmt: case T_VacuumStmt: + case T_RepackStmt: { /* * These commands write WAL, so they're not strictly @@ -862,6 +863,10 @@ standard_ProcessUtility(PlannedStmt *pstmt, ExecVacuum(pstate, (VacuumStmt *) parsetree, isTopLevel); break; + case T_RepackStmt: + repack(pstate, (RepackStmt *) parsetree, isTopLevel); + break; + case T_ExplainStmt: ExplainQuery(pstate, (ExplainStmt *) parsetree, params, dest); break; @@ -2869,6 +2874,10 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_ANALYZE; break; + case T_RepackStmt: + tag = CMDTAG_REPACK; + break; + case T_ExplainStmt: tag = CMDTAG_EXPLAIN; break; diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index e9096a8849..fdfb63e3ba 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -268,6 +268,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) cmdtype = PROGRESS_COMMAND_ANALYZE; else if (pg_strcasecmp(cmd, "CLUSTER") == 0) cmdtype = PROGRESS_COMMAND_CLUSTER; + else if (pg_strcasecmp(cmd, "REPACK") == 0) + cmdtype = PROGRESS_COMMAND_REPACK; else if (pg_strcasecmp(cmd, "CREATE INDEX") == 0) cmdtype = PROGRESS_COMMAND_CREATE_INDEX; else if (pg_strcasecmp(cmd, "BASEBACKUP") == 0) diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index eb8bc12872..5c32ef1cfb 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -1223,7 +1223,7 @@ static const char *const sql_commands[] = { "DELETE FROM", "DISCARD", "DO", "DROP", "END", "EXECUTE", "EXPLAIN", "FETCH", "GRANT", "IMPORT FOREIGN SCHEMA", "INSERT INTO", "LISTEN", "LOAD", "LOCK", "MERGE INTO", "MOVE", "NOTIFY", "PREPARE", - "REASSIGN", "REFRESH MATERIALIZED VIEW", "REINDEX", "RELEASE", + "REASSIGN", "REFRESH MATERIALIZED VIEW", "REINDEX", "RELEASE", "REPACK", "RESET", "REVOKE", "ROLLBACK", "SAVEPOINT", "SECURITY LABEL", "SELECT", "SET", "SHOW", "START", "TABLE", "TRUNCATE", "UNLISTEN", "UPDATE", "VACUUM", "VALUES", "WITH", @@ -4909,6 +4909,35 @@ match_previous_words(int pattern_id, COMPLETE_WITH_QUERY(Query_for_list_of_tablespaces); } +/* REPACK */ + else if (Matches("REPACK")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_clusterables); + else if (Matches("REPACK", "(*)")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_clusterables); + /* If we have REPACK <sth>, then add "USING" */ + else if (Matches("REPACK", MatchAnyExcept("("))) + COMPLETE_WITH("USING INDEX"); + /* If we have REPACK (*) <sth>, then add "USING" */ + else if (Matches("REPACK", "(*)", MatchAny)) + COMPLETE_WITH("USING INDEX"); + /* If we have REPACK <sth> USING, then add the index as well */ + else if (Matches("REPACK", MatchAny, "USING", "INDEX")) + { + set_completion_reference(prev3_wd); + COMPLETE_WITH_SCHEMA_QUERY(Query_for_index_of_table); + } + else if (HeadMatches("REPACK", "(*") && + !HeadMatches("REPACK", "(*)")) + { + /* + * This fires if we're in an unfinished parenthesized option list. + * get_previous_words treats a completed parenthesized option list as + * one word, so the above test is correct. + */ + if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) + COMPLETE_WITH("VERBOSE"); + } + /* SECURITY LABEL */ else if (Matches("SECURITY")) COMPLETE_WITH("LABEL"); diff --git a/src/include/commands/cluster.h b/src/include/commands/cluster.h index 60088a64cb..c2976905e4 100644 --- a/src/include/commands/cluster.h +++ b/src/include/commands/cluster.h @@ -31,10 +31,27 @@ typedef struct ClusterParams bits32 options; /* bitmask of CLUOPT_* */ } ClusterParams; +/* + * cluster.c currently implements three nearly identical commands: CLUSTER, + * VACUUM FULL and REPACK. Where needed, use this enumeration to distinguish + * which of these commands is being executed. + * + * Remove this stuff when removing the (now deprecated) CLUSTER and VACUUM + * FULL commands. + */ +typedef enum ClusterCommand +{ + CLUSTER_COMMAND_CLUSTER, + CLUSTER_COMMAND_REPACK, + CLUSTER_COMMAND_VACUUM +} ClusterCommand; + extern void cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel); -extern void cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params); +extern void cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params, + ClusterCommand cmd); extern void check_index_is_clusterable(Relation OldHeap, Oid indexOid, - LOCKMODE lockmode); + LOCKMODE lockmode, + ClusterCommand cmd); extern void mark_index_clustered(Relation rel, Oid indexOid, bool is_internal); extern Oid make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, Oid NewAccessMethod, @@ -48,4 +65,5 @@ extern void finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, MultiXactId cutoffMulti, char newrelpersistence); +extern void repack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); #endif /* CLUSTER_H */ diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 7c736e7b03..931dab215e 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -56,28 +56,51 @@ #define PROGRESS_ANALYZE_PHASE_COMPUTE_EXT_STATS 4 #define PROGRESS_ANALYZE_PHASE_FINALIZE_ANALYZE 5 -/* Progress parameters for cluster */ -#define PROGRESS_CLUSTER_COMMAND 0 -#define PROGRESS_CLUSTER_PHASE 1 -#define PROGRESS_CLUSTER_INDEX_RELID 2 -#define PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED 3 -#define PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN 4 -#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5 -#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6 -#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7 - -/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */ -#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1 -#define PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP 2 -#define PROGRESS_CLUSTER_PHASE_SORT_TUPLES 3 -#define PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP 4 -#define PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES 5 -#define PROGRESS_CLUSTER_PHASE_REBUILD_INDEX 6 -#define PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP 7 +/* + * Progress parameters for REPACK. + * + * CAUTION: These values are also used by CLUSTER. When enhancing REPACK, add + * the new values at the end of the list to avoid renumbering. + */ +#define PROGRESS_REPACK_COMMAND 0 +#define PROGRESS_REPACK_PHASE 1 +#define PROGRESS_REPACK_INDEX_RELID 2 +#define PROGRESS_REPACK_HEAP_TUPLES_SCANNED 3 +#define PROGRESS_REPACK_HEAP_TUPLES_WRITTEN 4 +#define PROGRESS_REPACK_TOTAL_HEAP_BLKS 5 +#define PROGRESS_REPACK_HEAP_BLKS_SCANNED 6 +#define PROGRESS_REPACK_INDEX_REBUILD_COUNT 7 + +/* + * Phases of repack (as advertised via PROGRESS_REPACK_PHASE). + * + * CAUTION: These values are also used by CLUSTER. When enhancing REPACK, add + * the new values at the end of the list to avoid renumbering. + */ +#define PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP 1 +#define PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP 2 +#define PROGRESS_REPACK_PHASE_SORT_TUPLES 3 +#define PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP 4 +#define PROGRESS_REPACK_PHASE_SWAP_REL_FILES 5 +#define PROGRESS_REPACK_PHASE_REBUILD_INDEX 6 +#define PROGRESS_REPACK_PHASE_FINAL_CLEANUP 7 + +/* Commands of PROGRESS_REPACK */ +#define PROGRESS_REPACK_COMMAND_REPACK 1 + +/* + * Progress parameters for cluster. + * + * Although we need to report REPACK and CLUSTER in separate views, the + * parameters and phases of CLUSTER are a subset of those of REPACK. Therefore + * we just use the appropriate values defined for REPACK above instead of + * defining a separate set of constants here. + */ /* Commands of PROGRESS_CLUSTER */ #define PROGRESS_CLUSTER_COMMAND_CLUSTER 1 #define PROGRESS_CLUSTER_COMMAND_VACUUM_FULL 2 +#define PROGRESS_CLUSTER_COMMAND_REPACK 3 /* Progress parameters for CREATE INDEX */ /* 3, 4 and 5 reserved for "waitfor" metrics */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 8dd421fa0e..f8225722fd 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3903,6 +3903,19 @@ typedef struct ClusterStmt List *params; /* list of DefElem nodes */ } ClusterStmt; +/* ---------------------- + * Repack Statement + * ---------------------- + */ +typedef struct RepackStmt +{ + NodeTag type; + RangeVar *relation; /* relation being repacked */ + char *indexname; /* order tuples by this index */ + List *params; /* list of DefElem nodes */ +} RepackStmt; + + /* ---------------------- * Vacuum and Analyze Statements * diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 40cf090ce6..0932d6fce5 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -373,6 +373,7 @@ PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("repack", REPACK, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index d250a714d5..cceb312f2b 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -196,6 +196,7 @@ PG_CMDTAG(CMDTAG_REASSIGN_OWNED, "REASSIGN OWNED", false, false, false) PG_CMDTAG(CMDTAG_REFRESH_MATERIALIZED_VIEW, "REFRESH MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_REINDEX, "REINDEX", true, false, false) PG_CMDTAG(CMDTAG_RELEASE, "RELEASE", false, false, false) +PG_CMDTAG(CMDTAG_REPACK, "REPACK", false, false, false) PG_CMDTAG(CMDTAG_RESET, "RESET", false, false, false) PG_CMDTAG(CMDTAG_REVOKE, "REVOKE", true, false, false) PG_CMDTAG(CMDTAG_REVOKE_ROLE, "REVOKE ROLE", false, false, false) diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h index dda813ab40..da3d14bb97 100644 --- a/src/include/utils/backend_progress.h +++ b/src/include/utils/backend_progress.h @@ -25,6 +25,7 @@ typedef enum ProgressCommandType PROGRESS_COMMAND_VACUUM, PROGRESS_COMMAND_ANALYZE, PROGRESS_COMMAND_CLUSTER, + PROGRESS_COMMAND_REPACK, PROGRESS_COMMAND_CREATE_INDEX, PROGRESS_COMMAND_BASEBACKUP, PROGRESS_COMMAND_COPY, diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out index 4d40a6809a..ed7df29b8e 100644 --- a/src/test/regress/expected/cluster.out +++ b/src/test/regress/expected/cluster.out @@ -254,6 +254,120 @@ ORDER BY 1; clstr_tst_pkey (3 rows) +-- REPACK handles individual tables identically to CLUSTER, but it's worth +-- checking if it handles table hierarchies identically as well. +REPACK clstr_tst USING INDEX clstr_tst_c; +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 2'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + a | b | c | substring | length +----+-----+------------------+--------------------------------+-------- + 10 | 14 | catorce | | + 18 | 5 | cinco | | + 9 | 4 | cuatro | | + 26 | 19 | diecinueve | | + 12 | 18 | dieciocho | | + 30 | 16 | dieciseis | | + 24 | 17 | diecisiete | | + 2 | 10 | diez | | + 23 | 12 | doce | | + 11 | 2 | dos | | + 25 | 9 | nueve | | + 31 | 8 | ocho | | + 1 | 11 | once | | + 28 | 15 | quince | | + 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 29 | 7 | siete | | + 15 | 13 | trece | | + 22 | 30 | treinta | | + 17 | 32 | treinta y dos | | + 3 | 31 | treinta y uno | | + 5 | 3 | tres | | + 20 | 1 | uno | | + 6 | 20 | veinte | | + 14 | 25 | veinticinco | | + 21 | 24 | veinticuatro | | + 4 | 22 | veintidos | | + 19 | 29 | veintinueve | | + 16 | 28 | veintiocho | | + 27 | 26 | veintiseis | | + 13 | 27 | veintisiete | | + 7 | 23 | veintitres | | + 8 | 21 | veintiuno | | + 0 | 100 | in child table | | + 0 | 100 | in child table 2 | | +(34 rows) + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); +ERROR: insert or update on table "clstr_tst" violates foreign key constraint "clstr_tst_con" +DETAIL: Key (b)=(1111) is not present in table "clstr_tst_s". +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + conname +---------------------- + clstr_tst_a_not_null + clstr_tst_con + clstr_tst_pkey +(3 rows) + +-- Yet another code path: REPACK w/o index. +REPACK clstr_tst USING INDEX clstr_tst_c; +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 3'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + a | b | c | substring | length +----+-----+------------------+--------------------------------+-------- + 10 | 14 | catorce | | + 18 | 5 | cinco | | + 9 | 4 | cuatro | | + 26 | 19 | diecinueve | | + 12 | 18 | dieciocho | | + 30 | 16 | dieciseis | | + 24 | 17 | diecisiete | | + 2 | 10 | diez | | + 23 | 12 | doce | | + 11 | 2 | dos | | + 25 | 9 | nueve | | + 31 | 8 | ocho | | + 1 | 11 | once | | + 28 | 15 | quince | | + 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 29 | 7 | siete | | + 15 | 13 | trece | | + 22 | 30 | treinta | | + 17 | 32 | treinta y dos | | + 3 | 31 | treinta y uno | | + 5 | 3 | tres | | + 20 | 1 | uno | | + 6 | 20 | veinte | | + 14 | 25 | veinticinco | | + 21 | 24 | veinticuatro | | + 4 | 22 | veintidos | | + 19 | 29 | veintinueve | | + 16 | 28 | veintiocho | | + 27 | 26 | veintiseis | | + 13 | 27 | veintisiete | | + 7 | 23 | veintitres | | + 8 | 21 | veintiuno | | + 0 | 100 | in child table | | + 0 | 100 | in child table 2 | | + 0 | 100 | in child table 3 | | +(35 rows) + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); +ERROR: insert or update on table "clstr_tst" violates foreign key constraint "clstr_tst_con" +DETAIL: Key (b)=(1111) is not present in table "clstr_tst_s". +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + conname +---------------------- + clstr_tst_a_not_null + clstr_tst_con + clstr_tst_pkey +(3 rows) + SELECT relname, relkind, EXISTS(SELECT 1 FROM pg_class WHERE oid = c.reltoastrelid) AS hastoast FROM pg_class c WHERE relname LIKE 'clstr_tst%' ORDER BY relname; @@ -381,6 +495,35 @@ SELECT * FROM clstr_1; 2 (2 rows) +-- REPACK w/o argument performs no ordering, so we can only check which tables +-- have the relfilenode changed. +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_old AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); +SET SESSION AUTHORIZATION regress_clstr_user; +SET client_min_messages = ERROR; -- order of "skipping" warnings may vary +REPACK; +RESET client_min_messages; +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_new AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); +-- Do the actual comparison. Unlike CLUSTER, clstr_3 should have been +-- processed because there is nothing like clustering index here. +SELECT o.relname FROM relnodes_old o +JOIN relnodes_new n ON o.relname = n.relname +WHERE o.relfilenode <> n.relfilenode +ORDER BY o.relname; + relname +--------- + clstr_1 + clstr_3 +(2 rows) + +SET SESSION AUTHORIZATION regress_clstr_user; -- Test MVCC-safety of cluster. There isn't much we can do to verify the -- results with a single backend... CREATE TABLE clustertest (key int PRIMARY KEY); @@ -495,6 +638,43 @@ ALTER TABLE clstrpart SET WITHOUT CLUSTER; ERROR: cannot mark index clustered in partitioned table ALTER TABLE clstrpart CLUSTER ON clstrpart_idx; ERROR: cannot mark index clustered in partitioned table +-- Check that REPACK sets new relfilenodes: it should process exactly the same +-- tables as CLUSTER did. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart USING INDEX clstrpart_idx; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + relname | level | relkind | ?column? +-------------+-------+---------+---------- + clstrpart | 0 | p | t + clstrpart1 | 1 | p | t + clstrpart11 | 2 | r | f + clstrpart12 | 2 | p | t + clstrpart2 | 1 | r | f + clstrpart3 | 1 | p | t + clstrpart33 | 2 | r | f +(7 rows) + +-- And finally the same for REPACK w/o index. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + relname | level | relkind | ?column? +-------------+-------+---------+---------- + clstrpart | 0 | p | t + clstrpart1 | 1 | p | t + clstrpart11 | 2 | r | f + clstrpart12 | 2 | p | t + clstrpart2 | 1 | r | f + clstrpart3 | 1 | p | t + clstrpart33 | 2 | r | f +(7 rows) + DROP TABLE clstrpart; -- Ownership of partitions is checked CREATE TABLE ptnowner(i int unique) PARTITION BY LIST (i); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 5baba8d39f..d678256816 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2041,6 +2041,33 @@ pg_stat_progress_create_index| SELECT s.pid, s.param15 AS partitions_done FROM (pg_stat_get_progress_info('CREATE INDEX'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); +pg_stat_progress_repack| SELECT s.pid, + s.datid, + d.datname, + s.relid, + CASE s.param1 + WHEN 1 THEN 'REPACK'::text + ELSE NULL::text + END AS command, + CASE s.param2 + WHEN 0 THEN 'initializing'::text + WHEN 1 THEN 'seq scanning heap'::text + WHEN 2 THEN 'index scanning heap'::text + WHEN 3 THEN 'sorting tuples'::text + WHEN 4 THEN 'writing new heap'::text + WHEN 5 THEN 'swapping relation files'::text + WHEN 6 THEN 'rebuilding index'::text + WHEN 7 THEN 'performing final cleanup'::text + ELSE NULL::text + END AS phase, + (s.param3)::oid AS repack_index_relid, + s.param4 AS heap_tuples_scanned, + s.param5 AS heap_tuples_written, + s.param6 AS heap_blks_total, + s.param7 AS heap_blks_scanned, + s.param8 AS index_rebuild_count + FROM (pg_stat_get_progress_info('REPACK'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) + LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_vacuum| SELECT s.pid, s.datid, d.datname, diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql index b7115f8610..e348e26fbf 100644 --- a/src/test/regress/sql/cluster.sql +++ b/src/test/regress/sql/cluster.sql @@ -76,6 +76,33 @@ INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass ORDER BY 1; +-- REPACK handles individual tables identically to CLUSTER, but it's worth +-- checking if it handles table hierarchies identically as well. +REPACK clstr_tst USING INDEX clstr_tst_c; + +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 2'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); + +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + +-- Yet another code path: REPACK w/o index. +REPACK clstr_tst USING INDEX clstr_tst_c; + +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 3'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); + +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + SELECT relname, relkind, EXISTS(SELECT 1 FROM pg_class WHERE oid = c.reltoastrelid) AS hastoast @@ -159,6 +186,34 @@ INSERT INTO clstr_1 VALUES (1); CLUSTER clstr_1; SELECT * FROM clstr_1; +-- REPACK w/o argument performs no ordering, so we can only check which tables +-- have the relfilenode changed. +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_old AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); + +SET SESSION AUTHORIZATION regress_clstr_user; +SET client_min_messages = ERROR; -- order of "skipping" warnings may vary +REPACK; +RESET client_min_messages; + +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_new AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); + +-- Do the actual comparison. Unlike CLUSTER, clstr_3 should have been +-- processed because there is nothing like clustering index here. +SELECT o.relname FROM relnodes_old o +JOIN relnodes_new n ON o.relname = n.relname +WHERE o.relfilenode <> n.relfilenode +ORDER BY o.relname; + +SET SESSION AUTHORIZATION regress_clstr_user; + -- Test MVCC-safety of cluster. There isn't much we can do to verify the -- results with a single backend... @@ -229,6 +284,24 @@ SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM o CLUSTER clstrpart; ALTER TABLE clstrpart SET WITHOUT CLUSTER; ALTER TABLE clstrpart CLUSTER ON clstrpart_idx; + +-- Check that REPACK sets new relfilenodes: it should process exactly the same +-- tables as CLUSTER did. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart USING INDEX clstrpart_idx; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + +-- And finally the same for REPACK w/o index. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + DROP TABLE clstrpart; -- Ownership of partitions is checked diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 64c6bf7a89..00d0fc7296 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -408,6 +408,7 @@ ClientCertName ClientConnectionInfo ClientData ClientSocket +ClusterCommand ClonePtrType ClosePortalStmt ClosePtrType @@ -2452,6 +2453,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepOriginId +RepackStmt ReparameterizeForeignPathByChild_function ReplaceVarsFromTargetList_context ReplaceVarsNoMatchOption -- 2.43.5 ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: why there is not VACUUM FULL CONCURRENTLY? @ 2025-02-19 17:40 Marcos Pegoraro <[email protected]> parent: Antonin Houska <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Marcos Pegoraro @ 2025-02-19 17:40 UTC (permalink / raw) To: Antonin Houska <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Michael Banck <[email protected]>; Junwang Zhao <[email protected]>; Kirill Reshke <[email protected]>; Pavel Stehule <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers You mentioned fillfactor only on cluster notes, would be good to mention it on refsynopsisdiv, I think. + <para> + <command>REPACK</command> reclaims storage occupied by dead + tuples. Unlike <command>VACUUM</command>, it does so by rewriting the + entire contents of the table specified + by <replaceable class="parameter">table_name</replaceable> into a new disk - file with no extra space, allowing unused space to be returned to the + file with no extra space, obeying fillfactor settings, allowing unused space to be returned to the + operating + system. + </para> regards Marcos Em qua., 19 de fev. de 2025 Ã s 14:08, Antonin Houska <[email protected]> escreveu: > Alvaro Herrera <[email protected]> wrote: > > > On 2025-Jan-30, Michael Banck wrote: > > > > > > I haven't addressed the problem of a new command yet - for that I'd > like to > > > > see some sort of consensus, so that I do not have to do all the > related > > > > changes many times. > > > > > > Well, looks like this patch-set is blocked on the bikeshedding part? > > > > > > Somebody should call a shot here, then. > > > > A bunch of people discussed this patch in today's developer meeting in > > Brussels. There's pretty much a consensus on using the verb REPACK > > CONCURRENTLY for this new command -- where unadorned REPACK would be > > VACUUM FULL, and we'd have something like REPACK WITH INDEX or maybe > > REPACK USING INDEX to take the CLUSTER place. > > This is a patch that adds the REPACK command (w/o CONCURRENTLY). I'll > incorporate it into the patch series but it'd be great if this part was a > little bit stable before I start to rebase the depending patches. Thanks. > > -- > Antonin Houska > Web: https://www.cybertec-postgresql.com > > > -- > > *E-Mail Disclaimer* > Der Inhalt dieser E-Mail ist ausschliesslich fuer den > bezeichneten Adressaten bestimmt. Wenn Sie nicht der vorgesehene Adressat > dieser E-Mail oder dessen Vertreter sein sollten, so beachten Sie bitte, > dass jede Form der Kenntnisnahme, Veroeffentlichung, Vervielfaeltigung > oder > Weitergabe des Inhalts dieser E-Mail unzulaessig ist. Wir bitten Sie, sich > in diesem Fall mit dem Absender der E-Mail in Verbindung zu setzen. > > *CONFIDENTIALITY NOTICE & DISCLAIMER > *This message and any attachment are > confidential and may be privileged or otherwise protected from disclosure > and solely for the use of the person(s) or entity to whom it is intended. > If you have received this message in error and are not the intended > recipient, please notify the sender immediately and delete this message > and > any attachment from your system. If you are not the intended recipient, be > advised that any use of this message is prohibited and may be unlawful, > and > you must not copy this message or attachment or disclose the contents to > any other person. > ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: Flush some statistics within running transactions @ 2026-03-16 06:26 Michael Paquier <[email protected]> 0 siblings, 1 reply; 58+ messages in thread From: Michael Paquier @ 2026-03-16 06:26 UTC (permalink / raw) To: Bertrand Drouvot <[email protected]>; +Cc: Sami Imseih <[email protected]>; Fujii Masao <[email protected]>; [email protected]; Zsolt Parragi <[email protected]> On Tue, Feb 24, 2026 at 12:01:30PM +0000, Bertrand Drouvot wrote: > Though I don't think that adresses Michael's concern: "main worries are > mainly around 1), I guess, with the new SIGALRM handler requirements for all > auxiliary processes" in [1]. FWIW, I am still concerned about that, and I have pondered about what we could do here. While reviewing the existing code, one thing that I have noticed we could do is rely on the existing interface of pgstat_report_stat() without changing the existing callers, and not touching at all the flush callbacks. If we begin to require the "force" mode when the routine the called inside a transaction block, things seem to work pretty smoothly in combination with a stats kind property that allows the stats data to be flushed if we are inside a transaction while a report happens. And it is possible to enforce checks inside pgstat_report_stat() as well. So please find attached my shot at that: - Introduction of a new system function called pg_stat_report(), based on a procsignal that gives a way to signal backends for a stats update, reusing the existing code where we only do flushes when idle and not in a transaction. - Property that tracks under which contexts the reports are allowed. Here I have decided to stick with simple, as in only allowing IO and WAL stats to be flushed if we are inside a transaction. Using that, I have done a few tests with three backends: - One with a long-running transaction. - One that periodically triggers the reports. - One that looks at IO and WAL stat. And the third session is able to get refreshes for both of these stats kinds, while the other stats remain the same. Note that this is a WIP, which is check-world stable. One thing that sticks a bit in mind now is that perhaps we should not allow the function for auxiliary processes at all. A second thing is the requirement of allowing partial flushes at the end of the report path, which is OK because the variable-sized stats can have pending data. Perhaps we should just have pgstat_flush_pending_entries() provide a correct status in line with the property set in a stats kind when we try a flush while in a transaction. Thoughts or tomatoes? -- Michael From c5aeb083265efbd6041ed3868b669997d4430760 Mon Sep 17 00:00:00 2001 From: Michael Paquier <[email protected]> Date: Mon, 16 Mar 2026 15:12:24 +0900 Subject: [PATCH] Add support for transient stats updates This introduces a function able to push stats updates, with a new stats kind property to allow stats to be updated while in a transaction. --- src/include/catalog/pg_proc.dat | 7 +++ src/include/miscadmin.h | 1 + src/include/storage/procsignal.h | 1 + src/include/utils/pgstat_internal.h | 16 ++++++ src/backend/storage/ipc/procsignal.c | 16 ++++++ src/backend/tcop/postgres.c | 11 +++- src/backend/utils/activity/pgstat.c | 42 ++++++++++++-- src/backend/utils/adt/pgstatfuncs.c | 58 ++++++++++++++++++++ src/backend/utils/init/globals.c | 1 + src/test/regress/expected/misc_functions.out | 52 ++++++++++++++++++ src/test/regress/sql/misc_functions.sql | 29 ++++++++++ doc/src/sgml/func/func-admin.sgml | 16 ++++++ 12 files changed, 242 insertions(+), 8 deletions(-) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 361e2cfffebe..85869154657a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -8692,6 +8692,13 @@ prosrc => 'pg_log_backend_memory_contexts', proacl => '{POSTGRES=X}' }, +# request an update of statistics +{ oid => '8789', descr => 'have the specified backend push a pgstats update', + proname => 'pg_stat_report', provolatile => 'v', + prorettype => 'bool', proargtypes => 'int4', + prosrc => 'pg_stat_report', + proacl => '{POSTGRES=X}' }, + # non-persistent series generator { oid => '1066', descr => 'non-persistent series generator', proname => 'generate_series', prorows => '1000', diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index f16f35659b9b..a245cdd79d17 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -94,6 +94,7 @@ extern PGDLLIMPORT volatile sig_atomic_t IdleInTransactionSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t TransactionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t IdleSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t ProcSignalBarrierPending; +extern PGDLLIMPORT volatile sig_atomic_t ProcSignalStatsUpdatePending; extern PGDLLIMPORT volatile sig_atomic_t LogMemoryContextPending; extern PGDLLIMPORT volatile sig_atomic_t IdleStatsUpdateTimeoutPending; diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 348fba53a931..d2e60403ad52 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -34,6 +34,7 @@ typedef enum PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ PROCSIG_WALSND_INIT_STOPPING, /* ask walsenders to prepare for shutdown */ PROCSIG_BARRIER, /* global barrier interrupt */ + PROCSIG_STATS_UPDATE, /* pgstats update */ PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ PROCSIG_RECOVERY_CONFLICT, /* backend is blocking recovery, check diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 9b8fbae00ed5..fa4cc4fe9c60 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -224,6 +224,17 @@ typedef struct PgStat_SubXactStatus PgStat_TableXactStatus *first; /* head of list for this subxact */ } PgStat_SubXactStatus; +/* + * Contexts related to the report of the statistics, defined as properties + * of PgStat_KindInfo.report_context. These define when a stats report is + * allowed depending on the stats kind and the context where + * pgstat_report_stat() is called. + */ + +/* report allowed while idle, outside a transaction (default) */ +#define PGSTAT_REPORT_IDLE 0x00 +/* report of stats data allowed within a transaction */ +#define PGSTAT_REPORT_TRANSACTION 0x01 /* * Metadata for a specific kind of statistics. @@ -251,6 +262,11 @@ typedef struct PgStat_KindInfo */ bool track_entry_count:1; + /* + * Contexts allowed for the report of this stats kind data. + */ + bits32 report_context; + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 7e017c8d53b5..e1bff2185933 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -490,6 +490,19 @@ HandleProcSignalBarrierInterrupt(void) /* latch will be set by procsignal_sigusr1_handler */ } +/* + * Handle receipt of an interrupt indicating that a stats update has been + * requested. This routine only gets called when PROCSIG_STATS_UPDATE is + * sent. + */ +static void +HandleProcSignalStatsUpdateInterrupt(void) +{ + InterruptPending = true; + ProcSignalStatsUpdatePending = true; + /* latch will be set by procsignal_sigusr1_handler */ +} + /* * Perform global barrier related interrupt checking. * @@ -694,6 +707,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_BARRIER)) HandleProcSignalBarrierInterrupt(); + if (CheckProcSignal(PROCSIG_STATS_UPDATE)) + HandleProcSignalStatsUpdateInterrupt(); + if (CheckProcSignal(PROCSIG_LOG_MEMORY_CONTEXT)) HandleLogMemoryContextInterrupt(); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index d01a09dd0c41..ddd57dfea780 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3555,12 +3555,17 @@ ProcessInterrupts(void) /* * If there are pending stats updates and we currently are truly idle - * (matching the conditions in PostgresMain(), report stats now. + * (matching the conditions in PostgresMain(), or if a status update has + * been requested, report stats now. */ if (IdleStatsUpdateTimeoutPending && - DoingCommandRead && !IsTransactionOrTransactionBlock()) + DoingCommandRead && !IsTransactionOrTransactionBlock() || + ProcSignalStatsUpdatePending) { - IdleStatsUpdateTimeoutPending = false; + if (IdleStatsUpdateTimeoutPending) + IdleStatsUpdateTimeoutPending = false; + if (ProcSignalStatsUpdatePending) + ProcSignalStatsUpdatePending = false; pgstat_report_stat(true); } diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 11bb71cad5ad..5521e96d0cae 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -436,6 +436,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .report_context = PGSTAT_REPORT_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), @@ -470,6 +471,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .report_context = PGSTAT_REPORT_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), @@ -698,8 +700,9 @@ pgstat_initialize(void) * a timeout after which to call pgstat_report_stat(true), but are not * required to do so. * - * Note that this is called only when not within a transaction, so it is fair - * to use transaction stop time as an approximation of current time. + * Note that when this is called only when not within a transaction, we use + * the transaction stop time as an approximation of current time. "force" + * is required when this is called within a transaction. */ long pgstat_report_stat(bool force) @@ -709,9 +712,14 @@ pgstat_report_stat(bool force) bool partial_flush; TimestampTz now; bool nowait; + bool is_xact = IsTransactionOrTransactionBlock(); pgstat_assert_is_up(); - Assert(!IsTransactionOrTransactionBlock()); + /* + * "force" is required if this routine is called inside a transaction + * block. + */ + Assert(!is_xact || force); /* "absorb" the forced flush even if there's nothing to flush */ if (pgStatForceNextFlush) @@ -789,6 +797,11 @@ pgstat_report_stat(bool force) if (!kind_info->flush_static_cb) continue; + /* Skip if this stats kind cannot be flushed in a transaction */ + if (is_xact && + (kind_info->report_context & PGSTAT_REPORT_TRANSACTION) == 0) + continue; + partial_flush |= kind_info->flush_static_cb(nowait); } } @@ -801,8 +814,11 @@ pgstat_report_stat(bool force) */ if (partial_flush) { - /* force should have prevented us from getting here */ - Assert(!force); + /* + * force should have prevented us from getting here, and partial + * flushes are accepted inside a transaction. + */ + Assert(!force || is_xact); /* remember since when stats have been pending */ if (pending_since == 0) @@ -1351,6 +1367,7 @@ pgstat_flush_pending_entries(bool nowait) { bool have_pending = false; dlist_node *cur = NULL; + bool is_xact = IsTransactionOrTransactionBlock(); /* * Need to be a bit careful iterating over the list of pending entries. @@ -1377,6 +1394,21 @@ pgstat_flush_pending_entries(bool nowait) Assert(!kind_info->fixed_amount); Assert(kind_info->flush_pending_cb != NULL); + /* Skip if this stats kind cannot be flushed while in a transaction */ + if (is_xact && + (kind_info->report_context & PGSTAT_REPORT_TRANSACTION) == 0) + { + have_pending = true; + + if (dlist_has_next(&pgStatPending, cur)) + next = dlist_next_node(&pgStatPending, cur); + else + next = NULL; + + cur = next; + continue; + } + /* flush the stats, if possible */ did_flush = kind_info->flush_pending_cb(entry_ref, nowait); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index bad5642d9c90..b27a5eb58e18 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -28,6 +28,7 @@ #include "replication/logicallauncher.h" #include "storage/proc.h" #include "storage/procarray.h" +#include "storage/procsignal.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/timestamp.h" @@ -2325,3 +2326,60 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objid)); } + +/* + * pg_stat_report + * Signal a backend or an auxiliary process to have it push an update + * of its statistics data. + * + * By default, only superusers are allowed to signal to log the memory + * contexts because allowing any users to issue this request at an unbounded + * rate would cause lots of log messages and which can lead to denial of + * service. Additional roles can be permitted with GRANT. + * + * On receipt of this signal, a backend or an auxiliary process sets the flag + * in the signal handler, which causes the next CHECK_FOR_INTERRUPTS() + * or process-specific interrupt handler to update their statistics. + */ +Datum +pg_stat_report(PG_FUNCTION_ARGS) +{ + int pid = PG_GETARG_INT32(0); + PGPROC *proc; + ProcNumber procNumber = INVALID_PROC_NUMBER; + + /* + * See if the process with given pid is a backend or an auxiliary process. + */ + proc = BackendPidGetProc(pid); + if (proc == NULL) + proc = AuxiliaryPidGetProc(pid); + + /* + * BackendPidGetProc() and AuxiliaryPidGetProc() return NULL if the pid + * isn't valid; but by the time we reach kill(), a process for which we + * get a valid proc here might have terminated on its own. This is OK, + * as at shutdown processes flush their stats. + */ + if (proc == NULL) + { + /* + * This is just a warning so a loop-through-resultset will not abort + * if one backend terminated on its own during the run. + */ + ereport(WARNING, + (errmsg("PID %d is not a PostgreSQL server process", pid))); + PG_RETURN_BOOL(false); + } + + procNumber = GetNumberFromPGProc(proc); + if (SendProcSignal(pid, PROCSIG_STATS_UPDATE, procNumber) < 0) + { + /* Again, just a warning to allow loops */ + ereport(WARNING, + (errmsg("could not send signal to process %d: %m", pid))); + PG_RETURN_BOOL(false); + } + + PG_RETURN_BOOL(true); +} diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index 36ad708b3602..22960ee3b27b 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -38,6 +38,7 @@ volatile sig_atomic_t IdleInTransactionSessionTimeoutPending = false; volatile sig_atomic_t TransactionTimeoutPending = false; volatile sig_atomic_t IdleSessionTimeoutPending = false; volatile sig_atomic_t ProcSignalBarrierPending = false; +volatile sig_atomic_t ProcSignalStatsUpdatePending = false; volatile sig_atomic_t LogMemoryContextPending = false; volatile sig_atomic_t IdleStatsUpdateTimeoutPending = false; volatile uint32 InterruptHoldoffCount = 0; diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 6c03b1a79d75..68a0e1e02bcc 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -397,6 +397,58 @@ REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM regress_log_memory; DROP ROLE regress_log_memory; -- +-- pg_stat_report +-- +-- check execution +SELECT pg_stat_report(pg_backend_pid()); + pg_stat_report +---------------- + t +(1 row) + +SELECT pg_stat_report(pid) FROM pg_stat_activity + WHERE backend_type = 'checkpointer'; + pg_stat_report +---------------- + t +(1 row) + +-- Check privileges +CREATE ROLE regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +-- Fails +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); +ERROR: permission denied for function pg_stat_report +RESET ROLE; +-- Access granted, then function works +GRANT EXECUTE ON FUNCTION pg_stat_report(integer) + TO regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); + pg_stat_report +---------------- + t +(1 row) + +RESET ROLE; +REVOKE EXECUTE ON FUNCTION pg_stat_report(integer) + FROM regress_stat_report; +DROP ROLE regress_stat_report; +-- -- Test some built-in SRFs -- -- The outputs of these are variable, so we can't just print their results diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 35b7983996c4..fe366904c9e8 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -154,6 +154,35 @@ REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) DROP ROLE regress_log_memory; +-- +-- pg_stat_report +-- + +-- check execution +SELECT pg_stat_report(pg_backend_pid()); +SELECT pg_stat_report(pid) FROM pg_stat_activity + WHERE backend_type = 'checkpointer'; + +-- Check privileges +CREATE ROLE regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- no +-- Fails +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); +RESET ROLE; +-- Access granted, then function works +GRANT EXECUTE ON FUNCTION pg_stat_report(integer) + TO regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- yes +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); +RESET ROLE; +REVOKE EXECUTE ON FUNCTION pg_stat_report(integer) + FROM regress_stat_report; +DROP ROLE regress_stat_report; + -- -- Test some built-in SRFs -- diff --git a/doc/src/sgml/func/func-admin.sgml b/doc/src/sgml/func/func-admin.sgml index 210b1118bdf7..114202c4fc19 100644 --- a/doc/src/sgml/func/func-admin.sgml +++ b/doc/src/sgml/func/func-admin.sgml @@ -220,6 +220,22 @@ </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_report</primary> + </indexterm> + <function>pg_log_backend_memory_contexts</function> ( <parameter>pid</parameter> <type>integer</type> ) + <returnvalue>boolean</returnvalue> + </para> + <para> + Requests to update the statistics computed by the backend with the + specified process ID. This function can send the request to + backends and auxiliary processes except logger, pushing an + update of the statistics data. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> -- 2.53.0 Attachments: [text/plain] 0001-Add-support-for-transient-stats-updates.patch (16.6K, ../../[email protected]/2-0001-Add-support-for-transient-stats-updates.patch) download | inline diff: From c5aeb083265efbd6041ed3868b669997d4430760 Mon Sep 17 00:00:00 2001 From: Michael Paquier <[email protected]> Date: Mon, 16 Mar 2026 15:12:24 +0900 Subject: [PATCH] Add support for transient stats updates This introduces a function able to push stats updates, with a new stats kind property to allow stats to be updated while in a transaction. --- src/include/catalog/pg_proc.dat | 7 +++ src/include/miscadmin.h | 1 + src/include/storage/procsignal.h | 1 + src/include/utils/pgstat_internal.h | 16 ++++++ src/backend/storage/ipc/procsignal.c | 16 ++++++ src/backend/tcop/postgres.c | 11 +++- src/backend/utils/activity/pgstat.c | 42 ++++++++++++-- src/backend/utils/adt/pgstatfuncs.c | 58 ++++++++++++++++++++ src/backend/utils/init/globals.c | 1 + src/test/regress/expected/misc_functions.out | 52 ++++++++++++++++++ src/test/regress/sql/misc_functions.sql | 29 ++++++++++ doc/src/sgml/func/func-admin.sgml | 16 ++++++ 12 files changed, 242 insertions(+), 8 deletions(-) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 361e2cfffebe..85869154657a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -8692,6 +8692,13 @@ prosrc => 'pg_log_backend_memory_contexts', proacl => '{POSTGRES=X}' }, +# request an update of statistics +{ oid => '8789', descr => 'have the specified backend push a pgstats update', + proname => 'pg_stat_report', provolatile => 'v', + prorettype => 'bool', proargtypes => 'int4', + prosrc => 'pg_stat_report', + proacl => '{POSTGRES=X}' }, + # non-persistent series generator { oid => '1066', descr => 'non-persistent series generator', proname => 'generate_series', prorows => '1000', diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index f16f35659b9b..a245cdd79d17 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -94,6 +94,7 @@ extern PGDLLIMPORT volatile sig_atomic_t IdleInTransactionSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t TransactionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t IdleSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t ProcSignalBarrierPending; +extern PGDLLIMPORT volatile sig_atomic_t ProcSignalStatsUpdatePending; extern PGDLLIMPORT volatile sig_atomic_t LogMemoryContextPending; extern PGDLLIMPORT volatile sig_atomic_t IdleStatsUpdateTimeoutPending; diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 348fba53a931..d2e60403ad52 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -34,6 +34,7 @@ typedef enum PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ PROCSIG_WALSND_INIT_STOPPING, /* ask walsenders to prepare for shutdown */ PROCSIG_BARRIER, /* global barrier interrupt */ + PROCSIG_STATS_UPDATE, /* pgstats update */ PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ PROCSIG_RECOVERY_CONFLICT, /* backend is blocking recovery, check diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 9b8fbae00ed5..fa4cc4fe9c60 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -224,6 +224,17 @@ typedef struct PgStat_SubXactStatus PgStat_TableXactStatus *first; /* head of list for this subxact */ } PgStat_SubXactStatus; +/* + * Contexts related to the report of the statistics, defined as properties + * of PgStat_KindInfo.report_context. These define when a stats report is + * allowed depending on the stats kind and the context where + * pgstat_report_stat() is called. + */ + +/* report allowed while idle, outside a transaction (default) */ +#define PGSTAT_REPORT_IDLE 0x00 +/* report of stats data allowed within a transaction */ +#define PGSTAT_REPORT_TRANSACTION 0x01 /* * Metadata for a specific kind of statistics. @@ -251,6 +262,11 @@ typedef struct PgStat_KindInfo */ bool track_entry_count:1; + /* + * Contexts allowed for the report of this stats kind data. + */ + bits32 report_context; + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 7e017c8d53b5..e1bff2185933 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -490,6 +490,19 @@ HandleProcSignalBarrierInterrupt(void) /* latch will be set by procsignal_sigusr1_handler */ } +/* + * Handle receipt of an interrupt indicating that a stats update has been + * requested. This routine only gets called when PROCSIG_STATS_UPDATE is + * sent. + */ +static void +HandleProcSignalStatsUpdateInterrupt(void) +{ + InterruptPending = true; + ProcSignalStatsUpdatePending = true; + /* latch will be set by procsignal_sigusr1_handler */ +} + /* * Perform global barrier related interrupt checking. * @@ -694,6 +707,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_BARRIER)) HandleProcSignalBarrierInterrupt(); + if (CheckProcSignal(PROCSIG_STATS_UPDATE)) + HandleProcSignalStatsUpdateInterrupt(); + if (CheckProcSignal(PROCSIG_LOG_MEMORY_CONTEXT)) HandleLogMemoryContextInterrupt(); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index d01a09dd0c41..ddd57dfea780 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3555,12 +3555,17 @@ ProcessInterrupts(void) /* * If there are pending stats updates and we currently are truly idle - * (matching the conditions in PostgresMain(), report stats now. + * (matching the conditions in PostgresMain(), or if a status update has + * been requested, report stats now. */ if (IdleStatsUpdateTimeoutPending && - DoingCommandRead && !IsTransactionOrTransactionBlock()) + DoingCommandRead && !IsTransactionOrTransactionBlock() || + ProcSignalStatsUpdatePending) { - IdleStatsUpdateTimeoutPending = false; + if (IdleStatsUpdateTimeoutPending) + IdleStatsUpdateTimeoutPending = false; + if (ProcSignalStatsUpdatePending) + ProcSignalStatsUpdatePending = false; pgstat_report_stat(true); } diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 11bb71cad5ad..5521e96d0cae 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -436,6 +436,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .report_context = PGSTAT_REPORT_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), @@ -470,6 +471,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .report_context = PGSTAT_REPORT_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), @@ -698,8 +700,9 @@ pgstat_initialize(void) * a timeout after which to call pgstat_report_stat(true), but are not * required to do so. * - * Note that this is called only when not within a transaction, so it is fair - * to use transaction stop time as an approximation of current time. + * Note that when this is called only when not within a transaction, we use + * the transaction stop time as an approximation of current time. "force" + * is required when this is called within a transaction. */ long pgstat_report_stat(bool force) @@ -709,9 +712,14 @@ pgstat_report_stat(bool force) bool partial_flush; TimestampTz now; bool nowait; + bool is_xact = IsTransactionOrTransactionBlock(); pgstat_assert_is_up(); - Assert(!IsTransactionOrTransactionBlock()); + /* + * "force" is required if this routine is called inside a transaction + * block. + */ + Assert(!is_xact || force); /* "absorb" the forced flush even if there's nothing to flush */ if (pgStatForceNextFlush) @@ -789,6 +797,11 @@ pgstat_report_stat(bool force) if (!kind_info->flush_static_cb) continue; + /* Skip if this stats kind cannot be flushed in a transaction */ + if (is_xact && + (kind_info->report_context & PGSTAT_REPORT_TRANSACTION) == 0) + continue; + partial_flush |= kind_info->flush_static_cb(nowait); } } @@ -801,8 +814,11 @@ pgstat_report_stat(bool force) */ if (partial_flush) { - /* force should have prevented us from getting here */ - Assert(!force); + /* + * force should have prevented us from getting here, and partial + * flushes are accepted inside a transaction. + */ + Assert(!force || is_xact); /* remember since when stats have been pending */ if (pending_since == 0) @@ -1351,6 +1367,7 @@ pgstat_flush_pending_entries(bool nowait) { bool have_pending = false; dlist_node *cur = NULL; + bool is_xact = IsTransactionOrTransactionBlock(); /* * Need to be a bit careful iterating over the list of pending entries. @@ -1377,6 +1394,21 @@ pgstat_flush_pending_entries(bool nowait) Assert(!kind_info->fixed_amount); Assert(kind_info->flush_pending_cb != NULL); + /* Skip if this stats kind cannot be flushed while in a transaction */ + if (is_xact && + (kind_info->report_context & PGSTAT_REPORT_TRANSACTION) == 0) + { + have_pending = true; + + if (dlist_has_next(&pgStatPending, cur)) + next = dlist_next_node(&pgStatPending, cur); + else + next = NULL; + + cur = next; + continue; + } + /* flush the stats, if possible */ did_flush = kind_info->flush_pending_cb(entry_ref, nowait); diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index bad5642d9c90..b27a5eb58e18 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -28,6 +28,7 @@ #include "replication/logicallauncher.h" #include "storage/proc.h" #include "storage/procarray.h" +#include "storage/procsignal.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/timestamp.h" @@ -2325,3 +2326,60 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objid)); } + +/* + * pg_stat_report + * Signal a backend or an auxiliary process to have it push an update + * of its statistics data. + * + * By default, only superusers are allowed to signal to log the memory + * contexts because allowing any users to issue this request at an unbounded + * rate would cause lots of log messages and which can lead to denial of + * service. Additional roles can be permitted with GRANT. + * + * On receipt of this signal, a backend or an auxiliary process sets the flag + * in the signal handler, which causes the next CHECK_FOR_INTERRUPTS() + * or process-specific interrupt handler to update their statistics. + */ +Datum +pg_stat_report(PG_FUNCTION_ARGS) +{ + int pid = PG_GETARG_INT32(0); + PGPROC *proc; + ProcNumber procNumber = INVALID_PROC_NUMBER; + + /* + * See if the process with given pid is a backend or an auxiliary process. + */ + proc = BackendPidGetProc(pid); + if (proc == NULL) + proc = AuxiliaryPidGetProc(pid); + + /* + * BackendPidGetProc() and AuxiliaryPidGetProc() return NULL if the pid + * isn't valid; but by the time we reach kill(), a process for which we + * get a valid proc here might have terminated on its own. This is OK, + * as at shutdown processes flush their stats. + */ + if (proc == NULL) + { + /* + * This is just a warning so a loop-through-resultset will not abort + * if one backend terminated on its own during the run. + */ + ereport(WARNING, + (errmsg("PID %d is not a PostgreSQL server process", pid))); + PG_RETURN_BOOL(false); + } + + procNumber = GetNumberFromPGProc(proc); + if (SendProcSignal(pid, PROCSIG_STATS_UPDATE, procNumber) < 0) + { + /* Again, just a warning to allow loops */ + ereport(WARNING, + (errmsg("could not send signal to process %d: %m", pid))); + PG_RETURN_BOOL(false); + } + + PG_RETURN_BOOL(true); +} diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index 36ad708b3602..22960ee3b27b 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -38,6 +38,7 @@ volatile sig_atomic_t IdleInTransactionSessionTimeoutPending = false; volatile sig_atomic_t TransactionTimeoutPending = false; volatile sig_atomic_t IdleSessionTimeoutPending = false; volatile sig_atomic_t ProcSignalBarrierPending = false; +volatile sig_atomic_t ProcSignalStatsUpdatePending = false; volatile sig_atomic_t LogMemoryContextPending = false; volatile sig_atomic_t IdleStatsUpdateTimeoutPending = false; volatile uint32 InterruptHoldoffCount = 0; diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out index 6c03b1a79d75..68a0e1e02bcc 100644 --- a/src/test/regress/expected/misc_functions.out +++ b/src/test/regress/expected/misc_functions.out @@ -397,6 +397,58 @@ REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) FROM regress_log_memory; DROP ROLE regress_log_memory; -- +-- pg_stat_report +-- +-- check execution +SELECT pg_stat_report(pg_backend_pid()); + pg_stat_report +---------------- + t +(1 row) + +SELECT pg_stat_report(pid) FROM pg_stat_activity + WHERE backend_type = 'checkpointer'; + pg_stat_report +---------------- + t +(1 row) + +-- Check privileges +CREATE ROLE regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- no + has_function_privilege +------------------------ + f +(1 row) + +-- Fails +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); +ERROR: permission denied for function pg_stat_report +RESET ROLE; +-- Access granted, then function works +GRANT EXECUTE ON FUNCTION pg_stat_report(integer) + TO regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- yes + has_function_privilege +------------------------ + t +(1 row) + +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); + pg_stat_report +---------------- + t +(1 row) + +RESET ROLE; +REVOKE EXECUTE ON FUNCTION pg_stat_report(integer) + FROM regress_stat_report; +DROP ROLE regress_stat_report; +-- -- Test some built-in SRFs -- -- The outputs of these are variable, so we can't just print their results diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql index 35b7983996c4..fe366904c9e8 100644 --- a/src/test/regress/sql/misc_functions.sql +++ b/src/test/regress/sql/misc_functions.sql @@ -154,6 +154,35 @@ REVOKE EXECUTE ON FUNCTION pg_log_backend_memory_contexts(integer) DROP ROLE regress_log_memory; +-- +-- pg_stat_report +-- + +-- check execution +SELECT pg_stat_report(pg_backend_pid()); +SELECT pg_stat_report(pid) FROM pg_stat_activity + WHERE backend_type = 'checkpointer'; + +-- Check privileges +CREATE ROLE regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- no +-- Fails +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); +RESET ROLE; +-- Access granted, then function works +GRANT EXECUTE ON FUNCTION pg_stat_report(integer) + TO regress_stat_report; +SELECT has_function_privilege('regress_stat_report', + 'pg_stat_report(integer)', 'EXECUTE'); -- yes +SET ROLE regress_stat_report; +SELECT pg_stat_report(pg_backend_pid()); +RESET ROLE; +REVOKE EXECUTE ON FUNCTION pg_stat_report(integer) + FROM regress_stat_report; +DROP ROLE regress_stat_report; + -- -- Test some built-in SRFs -- diff --git a/doc/src/sgml/func/func-admin.sgml b/doc/src/sgml/func/func-admin.sgml index 210b1118bdf7..114202c4fc19 100644 --- a/doc/src/sgml/func/func-admin.sgml +++ b/doc/src/sgml/func/func-admin.sgml @@ -220,6 +220,22 @@ </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_report</primary> + </indexterm> + <function>pg_log_backend_memory_contexts</function> ( <parameter>pid</parameter> <type>integer</type> ) + <returnvalue>boolean</returnvalue> + </para> + <para> + Requests to update the statistics computed by the backend with the + specified process ID. This function can send the request to + backends and auxiliary processes except logger, pushing an + update of the statistics data. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> -- 2.53.0 [application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc) download ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: Flush some statistics within running transactions @ 2026-03-16 09:20 Bertrand Drouvot <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 58+ messages in thread From: Bertrand Drouvot @ 2026-03-16 09:20 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Sami Imseih <[email protected]>; Fujii Masao <[email protected]>; [email protected]; Zsolt Parragi <[email protected]> Hi, On Mon, Mar 16, 2026 at 03:26:33PM +0900, Michael Paquier wrote: > On Tue, Feb 24, 2026 at 12:01:30PM +0000, Bertrand Drouvot wrote: > > Though I don't think that adresses Michael's concern: "main worries are > > mainly around 1), I guess, with the new SIGALRM handler requirements for all > > auxiliary processes" in [1]. > > FWIW, I am still concerned about that, and I have pondered about what > we could do here. While reviewing the existing code, one thing that I > have noticed we could do is rely on the existing interface of > pgstat_report_stat() without changing the existing callers, and not > touching at all the flush callbacks. If we begin to require the > "force" mode when the routine the called inside a transaction block, > things seem to work pretty smoothly in combination with a stats kind > property that allows the stats data to be flushed if we are inside a > transaction while a report happens. Yeah, "force" makes use of GetCurrentTimestamp() (and so we avoid a failed assertion that we would get if using GetCurrentTransactionStopTimestamp()). > So please find attached my shot at that: Thanks! > - Introduction of a new system function called pg_stat_report(), based > on a procsignal that gives a way to signal backends for a stats > update, reusing the existing code where we only do flushes when idle > and not in a transaction. > - Property that tracks under which contexts the reports are allowed. > Here I have decided to stick with simple, as in only allowing IO and > WAL stats to be flushed if we are inside a transaction. > > Using that, I have done a few tests with three backends: > - One with a long-running transaction. > - One that periodically triggers the reports. > - One that looks at IO and WAL stat. > And the third session is able to get refreshes for both of these stats > kinds, while the other stats remain the same. I did not look closely at the code but did some testing too. I confirm that pg_stat_io and pg_stat_wal are updated when pg_stat_report(<backend_pid>) is triggered. But the stats update is not visible if requested through pg_stat_get_backend_io(<same_backend_pid>) or pg_stat_get_backend_wal(<same_backend_pid>)). I guess that PGSTAT_KIND_BACKEND should also get the PGSTAT_REPORT_TRANSACTION report_context? > Note that this is a WIP, which is check-world stable. One thing that > sticks a bit in mind now is that perhaps we should not allow the > function for auxiliary processes at all. Why? > A second thing is the > requirement of allowing partial flushes at the end of the report path, > which is OK because the variable-sized stats can have pending data. Right. > Perhaps we should just have pgstat_flush_pending_entries() provide a > correct status in line with the property set in a stats kind when we > try a flush while in a transaction. The idea would be to avoid trying to flush stats that don't have pending entries? > Thoughts or tomatoes? That looks "simpler" that the previous proposal but who would be responsible to call pg_stat_report()? If that's the client responsabilty that kind of look weird to me. If that's the core, how would that be scheduled? I think that the end solution should prevent to find similar issues as 039549d70f6 fixed, without delegating to the client. Regards, -- Bertrand Drouvot PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: Flush some statistics within running transactions @ 2026-03-16 10:22 Michael Paquier <[email protected]> parent: Bertrand Drouvot <[email protected]> 0 siblings, 1 reply; 58+ messages in thread From: Michael Paquier @ 2026-03-16 10:22 UTC (permalink / raw) To: Bertrand Drouvot <[email protected]>; +Cc: Sami Imseih <[email protected]>; Fujii Masao <[email protected]>; [email protected]; Zsolt Parragi <[email protected]> On Mon, Mar 16, 2026 at 09:20:41AM +0000, Bertrand Drouvot wrote: > I did not look closely at the code but did some testing too. I confirm that > pg_stat_io and pg_stat_wal are updated when pg_stat_report(<backend_pid>) is > triggered. But the stats update is not visible if requested through > pg_stat_get_backend_io(<same_backend_pid>) or pg_stat_get_backend_wal(<same_backend_pid>)). > I guess that PGSTAT_KIND_BACKEND should also get the PGSTAT_REPORT_TRANSACTION > report_context? Yes, I guess that the transaction-level flag should be set as well for the backend stats, due to the fact that WAL and IO stats have it set. >> Note that this is a WIP, which is check-world stable. One thing that >> sticks a bit in mind now is that perhaps we should not allow the >> function for auxiliary processes at all. > > Why? I was feeling that there would be an issue with that, and I did not test it in details yet. For the checkpointer with a busy activity, the IO stats could be interesting to see in live. >> Perhaps we should just have pgstat_flush_pending_entries() provide a >> correct status in line with the property set in a stats kind when we >> try a flush while in a transaction. > > The idea would be to avoid trying to flush stats that don't have pending > entries? I had to work around the assert at the end of pgstat_report_stat(), to tell that under an xact the partial flushes were OK. I was wondering about keeping the end assertion based on solely "force" intact, making the flush routines return a status based on if we are in an xact. > That looks "simpler" that the previous proposal but who would be responsible to > call pg_stat_report()? If that's the client responsabilty that kind of look weird > to me. If that's the core, how would that be scheduled? I think that the > end solution should prevent to find similar issues as 039549d70f6 fixed, without > delegating to the client. TBH, I don't like the requirement of setting SIGALRM in all the places where we'd require it for the sake of this proposal, where historically we have never done that, copying a mechanism that already exists in the tree for procsigs, as the previous patch I posted proves we could reuse. It's also not clear to me what a correct frequency of the stat updates should be, and why it would make sense to force that in a GUC; we want to have some information from long-running transactions, where I fear that we will want modularity. A user-settable GUC would fit with this picture, but the requirement of planting the timeouts don't stick for me.. On top of that, I am not really convinced that there is a good reason to remove the existing stat report calls we have already planted in the tree for auxiliary processes, diverging from the stable branches where these exist. With more than one release already out with them, there is more benefit with potentially planting more strategic report calls where they could matter (as added in v18 for WAL senders as one example), when we find a requirement for them. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: Flush some statistics within running transactions @ 2026-03-16 23:42 Sami Imseih <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 58+ messages in thread From: Sami Imseih @ 2026-03-16 23:42 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; Fujii Masao <[email protected]>; [email protected]; Zsolt Parragi <[email protected]> Hi, I have spent some time thinking about Michael's feedback and I want to see if I can propose an alternate design that we can come to a consensus on. > I had to work around the assert at the end of pgstat_report_stat(), to > tell that under an xact the partial flushes were OK. I was wondering > about keeping the end assertion based on solely "force" intact, making > the flush routines return a status based on if we are in an xact. > > > That looks "simpler" that the previous proposal but who would be responsible to > > call pg_stat_report()? If that's the client responsibility that kind of look weird > > to me. If that's the core, how would that be scheduled? I think that the > > end solution should prevent to find similar issues as 039549d70f6 fixed, without > > delegating to the client. > > TBH, I don't like the requirement of setting SIGALRM in all the places > where we'd require it for the sake of this proposal, where > historically we have never done that, copying a mechanism that already > exists in the tree for procsigs, as the previous patch I posted proves > we could reuse. How about we add a timeout when a transaction goes idle? Currently we have IDLE_STATS_UPDATE_TIMEOUT, which is the timeout interval used when we have done more than one flush within a second. This only works when the session is idle (not in a transaction), and flushes everything. However, as I mentioned earlier in this thread, we can also schedule a flush that occurs when a transaction goes idle. This means when a transaction is started (BEGIN/START TRANSACTION) the proceeding idle-in-transaction can schedule a timeout of 10 seconds based on a new #define called PGSTAT_IDLE_TXN_INTERVAL. The timeout is not a fixed repeating interval. It is set on-demand each time the transaction enters idle-in-transaction state: first after the BEGIN, then again after each subsequent command completes. This is a trade-off: if a single command runs for a long time, no flush happens until it finishes and the transaction goes idle again. So, we still introduce a new timeout in this mechanism, but it does not need to be registered all over the place. So with this all the "safe to be flushed" stats will be flushed mid-transaction at some point, and all stats will be flushed at the end of a transaction. With that said, the only way I can conceive getting away from a a new timeout in this design is to track timestamp whenever we go into idle-intransaction, but I am not too comfy adding a GetCurrentTimestamp() there. -> if no action occurs at this point, we don't set another timeout > It's also not clear to me what a correct frequency of > the stat updates should be, and why it would make sense to force that > in a GUC; No GUC will be needed in what I am proposing. A 10s flush interval should be sufficient. > On top of that, I am not really convinced that there is a good reason > to remove the existing stat report calls we have already planted in > the tree for auxiliary processes, diverging from the stable branches > where these exist. The design being proposed keeps all the existing in-tact. So attached is a new proposal with tests and docs. In terms of test I fell back to the strategy used by Bertrand [0] with the INJECTION_POINT trick to reduce the flush interval. It does mean we keep a pg_sleep in the test, but I think we should not try to do anything different here. Also, with this design we can call the flush modes FLUSH_IN_TRANSACTION and FLUSH_AT_TXN_BOUNDARY. As the earlier proposals, the callbacks will still need to make the decision what stats to flush based on if it's called in-transaction or at boundary. [0] https://www.postgresql.org/message-id/aZbDYMrOkeCyIubO%40ip-10-97-1-34.eu-west-3.compute.internal -- Sami Imseih Amazon Web Services (AWS) Attachments: [application/octet-stream] v12-0001-Add-periodic-in-transaction-stats-flushing.patch (54.2K, ../../CAA5RZ0ssPraeF9gg8S8G7O25t90t8_Y2PU-jdNkZ42EiNH22CA@mail.gmail.com/2-v12-0001-Add-periodic-in-transaction-stats-flushing.patch) download | inline diff: From 5d75d93701bd268e7689791042405450297fede9 Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot <[email protected]> Date: Mon, 5 Jan 2026 09:41:39 +0000 Subject: [PATCH v12 1/1] Add periodic in-transaction stats flushing Some statistics, such as relation scan counts, blocks fetched/hit, WAL, IO, and SLRU stats, can safely be flushed to shared memory mid-transaction. Others, like tuple insert/update/delete counts, should only be flushed at transaction boundaries. Introduce a flush_mode field (FLUSH_IN_TRANSACTION and FLUSH_AT_TXN_BOUNDARY) fo each stats kind. FLUSH_IN_TRANSACTION will be flushed every PGSTAT_IDLE_TXN_INTERVAL (10s) while idle in transaction. The timeout is set each time the transaction goes idle between commands, not on a fixed repeating interval. It is disabled when the session goes idle after the transaction ends. --- doc/src/sgml/monitoring.sgml | 28 ++++- src/backend/replication/walsender.c | 8 +- src/backend/tcop/postgres.c | 43 +++++++ src/backend/utils/activity/pgstat.c | 109 ++++++++++++++--- src/backend/utils/activity/pgstat_backend.c | 6 +- src/backend/utils/activity/pgstat_bgwriter.c | 2 +- .../utils/activity/pgstat_checkpointer.c | 2 +- src/backend/utils/activity/pgstat_database.c | 6 +- src/backend/utils/activity/pgstat_function.c | 8 +- src/backend/utils/activity/pgstat_io.c | 6 +- src/backend/utils/activity/pgstat_relation.c | 98 ++++++++++++--- src/backend/utils/activity/pgstat_slru.c | 2 +- .../utils/activity/pgstat_subscription.c | 8 +- src/backend/utils/activity/pgstat_wal.c | 10 +- src/backend/utils/init/globals.c | 1 + src/backend/utils/init/postinit.c | 12 ++ src/include/miscadmin.h | 1 + src/include/pgstat.h | 8 ++ src/include/utils/pgstat_internal.h | 58 +++++++-- src/include/utils/timeout.h | 1 + .../test_custom_stats/t/001_custom_stats.pl | 59 +++++++++ .../test_custom_var_stats--1.0.sql | 10 ++ .../test_custom_stats/test_custom_var_stats.c | 114 +++++++++++++++++- src/test/modules/test_misc/meson.build | 1 + .../test_misc/t/011_in_transaction_stats.pl | 91 ++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 26 files changed, 611 insertions(+), 82 deletions(-) create mode 100644 src/test/modules/test_misc/t/011_in_transaction_stats.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 462019a972c..dcdeb9aabec 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -235,15 +235,31 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser When using the cumulative statistics views and functions to monitor collected data, it is important to realize that the information does not update instantaneously. Each individual server process flushes out - accumulated statistics to shared memory just before going idle, but not - more frequently than once per <varname>PGSTAT_MIN_INTERVAL</varname> - milliseconds (1 second unless altered while building the server); so a - query or transaction still in progress does not affect the displayed totals - and the displayed information lags behind actual activity. However, - current-query information collected by <varname>track_activities</varname> + accumulated statistics to shared memory just before going idle or + periodically during transactions, depending on whether the statistics + should only be flushed at a transaction boundary, as described below. + Current-query information collected by <varname>track_activities</varname> is always up-to-date. </para> + <para> + When a server process is about to become idle, it flushes all pending + statistics to shared memory, but not more frequently than once per + <varname>PGSTAT_MIN_INTERVAL</varname> milliseconds (1 second unless + altered while building the server). + </para> + + <para> + During explicit transaction blocks, statistics that can be flushed at any + time, such as relation scan counts, tuples returned and fetched, blocks + fetched and hit, as well as IO, WAL, SLRU, and backend statistics, are + periodically flushed to shared memory every + <varname>PGSTAT_IDLE_TXN_INTERVAL</varname> milliseconds (10 seconds + unless altered while building the server). This periodic flushing occurs + when the transaction goes idle between commands. It does not apply to + single statements running outside an explicit transaction block. + </para> + <para> Another important point is that when a server process is asked to display any of the accumulated statistics, accessed values are cached until the end diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 376ff46340d..24124d478e8 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -1993,8 +1993,8 @@ WalSndWaitForWal(XLogRecPtr loc) if (TimestampDifferenceExceeds(last_flush, now, WALSENDER_STATS_FLUSH_INTERVAL)) { - pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); + pgstat_flush_io(false, true); + (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO, true); last_flush = now; } @@ -3022,8 +3022,8 @@ WalSndLoop(WalSndSendDataCallback send_data) if (TimestampDifferenceExceeds(last_flush, now, WALSENDER_STATS_FLUSH_INTERVAL)) { - pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); + pgstat_flush_io(false, true); + (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO, true); last_flush = now; } diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index b3563113219..00436fee0de 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3565,6 +3565,12 @@ ProcessInterrupts(void) pgstat_report_stat(true); } + if (InTransactionStatsUpdateTimeoutPending) + { + InTransactionStatsUpdateTimeoutPending = false; + pgstat_report_in_transaction_stat(false); + } + if (ProcSignalBarrierPending) ProcessProcSignalBarrier(); @@ -4621,6 +4627,21 @@ PostgresMain(const char *dbname, const char *username) enable_timeout_after(IDLE_IN_TRANSACTION_SESSION_TIMEOUT, IdleInTransactionSessionTimeout); } + + /* + * Flush in-transaction stats. + */ + if (!get_timeout_active(IN_TRANSACTION_STATS_UPDATE_TIMEOUT)) + { + int stats_interval = PGSTAT_IDLE_TXN_INTERVAL; + +#ifdef USE_INJECTION_POINTS + if (IS_INJECTION_POINT_ATTACHED("in-transaction-stats-short-interval")) + stats_interval = 1000; +#endif + enable_timeout_after(IN_TRANSACTION_STATS_UPDATE_TIMEOUT, + stats_interval); + } } else if (IsTransactionOrTransactionBlock()) { @@ -4635,6 +4656,21 @@ PostgresMain(const char *dbname, const char *username) enable_timeout_after(IDLE_IN_TRANSACTION_SESSION_TIMEOUT, IdleInTransactionSessionTimeout); } + + /* + * Flush in-transaction stats. + */ + if (!get_timeout_active(IN_TRANSACTION_STATS_UPDATE_TIMEOUT)) + { + int stats_interval = PGSTAT_IDLE_TXN_INTERVAL; + +#ifdef USE_INJECTION_POINTS + if (IS_INJECTION_POINT_ATTACHED("in-transaction-stats-short-interval")) + stats_interval = 1000; +#endif + enable_timeout_after(IN_TRANSACTION_STATS_UPDATE_TIMEOUT, + stats_interval); + } } else { @@ -4650,6 +4686,13 @@ PostgresMain(const char *dbname, const char *username) if (notifyInterruptPending) ProcessNotifyInterrupt(false); + /* + * We are no longer in a transaction, so disable the + * in-transaction stats flush timeout if it was active. + */ + if (get_timeout_active(IN_TRANSACTION_STATS_UPDATE_TIMEOUT)) + disable_timeout(IN_TRANSACTION_STATS_UPDATE_TIMEOUT, false); + /* * Check if we need to report stats. If pgstat_report_stat() * decides it's too soon to flush out pending stats / lock diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 11bb71cad5a..70090565fee 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -187,7 +187,8 @@ static void pgstat_init_snapshot_fixed(void); static void pgstat_reset_after_failure(void); -static bool pgstat_flush_pending_entries(bool nowait); +static bool pgstat_flush_pending_entries(bool nowait, bool in_txn_only); +static bool pgstat_flush_fixed_stats(bool nowait, bool in_txn_only); static void pgstat_prep_snapshot(void); static void pgstat_build_snapshot(void); @@ -288,6 +289,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, + .flush_mode = FLUSH_IN_TRANSACTION, /* so pg_stat_database entries can be seen in all databases */ .accessed_across_databases = true, @@ -305,6 +307,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, + .flush_mode = FLUSH_IN_TRANSACTION, .shared_size = sizeof(PgStatShared_Relation), .shared_data_off = offsetof(PgStatShared_Relation, stats), @@ -321,6 +324,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, + .flush_mode = FLUSH_AT_TXN_BOUNDARY, .shared_size = sizeof(PgStatShared_Function), .shared_data_off = offsetof(PgStatShared_Function, stats), @@ -336,6 +340,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, + .flush_mode = FLUSH_AT_TXN_BOUNDARY, .accessed_across_databases = true, @@ -353,6 +358,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = true, + .flush_mode = FLUSH_AT_TXN_BOUNDARY, /* so pg_stat_subscription_stats entries can be seen in all databases */ .accessed_across_databases = true, @@ -370,6 +376,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = false, .write_to_file = false, + .flush_mode = FLUSH_IN_TRANSACTION, .accessed_across_databases = true, @@ -436,6 +443,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .flush_mode = FLUSH_IN_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), @@ -453,6 +461,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .flush_mode = FLUSH_IN_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, slru), .shared_ctl_off = offsetof(PgStat_ShmemControl, slru), @@ -470,6 +479,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .fixed_amount = true, .write_to_file = true, + .flush_mode = FLUSH_IN_TRANSACTION, .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), @@ -775,23 +785,11 @@ pgstat_report_stat(bool force) partial_flush = false; /* flush of variable-numbered stats tracked in pending entries list */ - partial_flush |= pgstat_flush_pending_entries(nowait); + partial_flush |= pgstat_flush_pending_entries(nowait, false); /* flush of other stats kinds */ if (pgstat_report_fixed) - { - for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++) - { - const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); - - if (!kind_info) - continue; - if (!kind_info->flush_static_cb) - continue; - - partial_flush |= kind_info->flush_static_cb(nowait); - } - } + partial_flush |= pgstat_flush_fixed_stats(nowait, false); last_flush = now; @@ -1293,7 +1291,8 @@ pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, uint64 objid, bool *creat if (entry_ref->pending == NULL) { - size_t entrysize = pgstat_get_kind_info(kind)->pending_size; + const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + size_t entrysize = kind_info->pending_size; Assert(entrysize != (size_t) -1); @@ -1345,9 +1344,15 @@ pgstat_delete_pending_entry(PgStat_EntryRef *entry_ref) /* * Flush out pending variable-numbered stats. + * + * If in_txn_only is true, only flushes FLUSH_IN_TRANSACTION entries. For entries + * that support it, the callback may flush only non-transactional fields. + * This is safe to call inside transactions. + * + * If in_txn_only is false, flushes all entries. */ static bool -pgstat_flush_pending_entries(bool nowait) +pgstat_flush_pending_entries(bool nowait, bool in_txn_only) { bool have_pending = false; dlist_node *cur = NULL; @@ -1372,13 +1377,29 @@ pgstat_flush_pending_entries(bool nowait) PgStat_Kind kind = key.kind; const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); bool did_flush; + bool is_partial_flush = false; dlist_node *next; Assert(!kind_info->fixed_amount); Assert(kind_info->flush_pending_cb != NULL); + /* Skip transactional stats if we're in in_txn_only mode */ + if (in_txn_only && kind_info->flush_mode == FLUSH_AT_TXN_BOUNDARY) + { + have_pending = true; + + if (dlist_has_next(&pgStatPending, cur)) + next = dlist_next_node(&pgStatPending, cur); + else + next = NULL; + + cur = next; + continue; + } + /* flush the stats, if possible */ - did_flush = kind_info->flush_pending_cb(entry_ref, nowait); + did_flush = kind_info->flush_pending_cb(entry_ref, nowait, + in_txn_only, &is_partial_flush); Assert(did_flush || nowait); @@ -1388,8 +1409,8 @@ pgstat_flush_pending_entries(bool nowait) else next = NULL; - /* if successfully flushed, remove entry */ - if (did_flush) + /* if successfull non-partial flush, remove entry */ + if (did_flush && !is_partial_flush) pgstat_delete_pending_entry(entry_ref); else have_pending = true; @@ -1402,6 +1423,33 @@ pgstat_flush_pending_entries(bool nowait) return have_pending; } +/* + * Flush fixed-amount stats. + * + * If in_txn_only is true, only flushes FLUSH_IN_TRANSACTION stats (safe inside transactions). + * If in_txn_only is false, flushes all stats with flush_static_cb. + */ +static bool +pgstat_flush_fixed_stats(bool nowait, bool in_txn_only) +{ + bool partial_flush = false; + + for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++) + { + const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + + if (!kind_info || !kind_info->flush_static_cb) + continue; + + /* Skip transactional stats if we're in in_txn_only mode */ + if (in_txn_only && kind_info->flush_mode == FLUSH_AT_TXN_BOUNDARY) + continue; + + partial_flush |= kind_info->flush_static_cb(nowait, in_txn_only); + } + + return partial_flush; +} /* ------------------------------------------------------------ * Helper / infrastructure functions @@ -2119,3 +2167,24 @@ assign_stats_fetch_consistency(int newval, void *extra) if (pgstat_fetch_consistency != newval) force_stats_snapshot_clear = true; } + +/* + * Flushes only FLUSH_IN_TRANSACTION stats using non-blocking locks. Transactional + * stats (FLUSH_AT_TXN_BOUNDARY) remain pending until transaction boundary. + * Safe to call inside transactions. + * + * This is called from a timeout handler every PGSTAT_IDLE_TXN_INTERVAL + * (10000ms) while idle in transaction, to ensure non-transactional stats + * (e.g., IO, backend) are flushed periodically even during long transactions. + */ +void +pgstat_report_in_transaction_stat(bool force) +{ + bool nowait = !force; + + pgstat_assert_is_up(); + + /* Flush stats outside of transaction boundary */ + pgstat_flush_pending_entries(nowait, true); + pgstat_flush_fixed_stats(nowait, true); +} diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index f2f8d3ff75f..3be8b2705b1 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -268,7 +268,7 @@ pgstat_flush_backend_entry_wal(PgStat_EntryRef *entry_ref) * if some statistics could not be flushed due to lock contention. */ bool -pgstat_flush_backend(bool nowait, bits32 flags) +pgstat_flush_backend(bool nowait, bits32 flags, bool in_txn_only) { PgStat_EntryRef *entry_ref; bool has_pending_data = false; @@ -311,9 +311,9 @@ pgstat_flush_backend(bool nowait, bits32 flags) * If some stats could not be flushed due to lock contention, return true. */ bool -pgstat_backend_flush_cb(bool nowait) +pgstat_backend_flush_cb(bool nowait, bool in_txn_only) { - return pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_ALL); + return pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_ALL, in_txn_only); } /* diff --git a/src/backend/utils/activity/pgstat_bgwriter.c b/src/backend/utils/activity/pgstat_bgwriter.c index ed2fd801189..1c5f0c3ec40 100644 --- a/src/backend/utils/activity/pgstat_bgwriter.c +++ b/src/backend/utils/activity/pgstat_bgwriter.c @@ -61,7 +61,7 @@ pgstat_report_bgwriter(void) /* * Report IO statistics */ - pgstat_flush_io(false); + pgstat_flush_io(false, true); } /* diff --git a/src/backend/utils/activity/pgstat_checkpointer.c b/src/backend/utils/activity/pgstat_checkpointer.c index 1f70194b7a7..2d89a082464 100644 --- a/src/backend/utils/activity/pgstat_checkpointer.c +++ b/src/backend/utils/activity/pgstat_checkpointer.c @@ -68,7 +68,7 @@ pgstat_report_checkpointer(void) /* * Report IO statistics */ - pgstat_flush_io(false); + pgstat_flush_io(false, true); } /* diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 933dcb5cae5..72d757a6631 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -435,7 +435,8 @@ pgstat_reset_database_timestamp(Oid dboid, TimestampTz ts) * false without flushing the entry. Otherwise returns true. */ bool -pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial) { PgStatShared_Database *sharedent; PgStat_StatDBEntry *pendingent; @@ -443,6 +444,9 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) pendingent = (PgStat_StatDBEntry *) entry_ref->pending; sharedent = (PgStatShared_Database *) entry_ref->shared_stats; + /* this is not a partial flush */ + *is_partial = false; + if (!pgstat_lock_entry(entry_ref, nowait)) return false; diff --git a/src/backend/utils/activity/pgstat_function.c b/src/backend/utils/activity/pgstat_function.c index e6b84283c6c..d51371d7967 100644 --- a/src/backend/utils/activity/pgstat_function.c +++ b/src/backend/utils/activity/pgstat_function.c @@ -190,14 +190,20 @@ pgstat_end_function_usage(PgStat_FunctionCallUsage *fcu, bool finalize) * false without flushing the entry. Otherwise returns true. */ bool -pgstat_function_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +pgstat_function_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial) { PgStat_FunctionCounts *localent; PgStatShared_Function *shfuncent; + Assert(!in_txn_only); + localent = (PgStat_FunctionCounts *) entry_ref->pending; shfuncent = (PgStatShared_Function *) entry_ref->shared_stats; + /* this is not a partial flush */ + *is_partial = false; + /* localent always has non-zero content */ if (!pgstat_lock_entry(entry_ref, nowait)) diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 28de24538dc..ec541bf3f78 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -172,9 +172,9 @@ pgstat_fetch_stat_io(void) * Simpler wrapper of pgstat_io_flush_cb() */ void -pgstat_flush_io(bool nowait) +pgstat_flush_io(bool nowait, bool in_txn_only) { - (void) pgstat_io_flush_cb(nowait); + (void) pgstat_io_flush_cb(nowait, in_txn_only); } /* @@ -186,7 +186,7 @@ pgstat_flush_io(bool nowait) * acquired. Otherwise, return false. */ bool -pgstat_io_flush_cb(bool nowait) +pgstat_io_flush_cb(bool nowait, bool in_txn_only) { LWLock *bktype_lock; PgStat_BktypeIO *bktype_shstats; diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index bc8c43b96aa..33359cf07b0 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -47,7 +47,19 @@ static void add_tabstat_xact_level(PgStat_TableStatus *pgstat_info, int nest_lev static void ensure_tabstat_xact_level(PgStat_TableStatus *pgstat_info); static void save_truncdrop_counters(PgStat_TableXactStatus *trans, bool is_drop); static void restore_truncdrop_counters(PgStat_TableXactStatus *trans); +static void flush_relation_in_transaction_stats(PgStat_StatTabEntry *tabentry, + PgStat_TableCounts *counts, bool in_txn_only); +/* + * Update database statistics with non-transactional stats. + */ +#define UPDATE_DATABASE_IN_TRANSACTION_STATS(dbentry, counts) \ + do { \ + (dbentry)->tuples_returned += (counts)->tuples_returned; \ + (dbentry)->tuples_fetched += (counts)->tuples_fetched; \ + (dbentry)->blocks_fetched += (counts)->blocks_fetched; \ + (dbentry)->blocks_hit += (counts)->blocks_hit; \ + } while (0) /* * Copy stats between relations. This is used for things like REINDEX @@ -267,8 +279,8 @@ pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, * is done -- which will likely vacuum many relations -- or until the * VACUUM command has processed all tables and committed. */ - pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); + pgstat_flush_io(false, true); + (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO, true); } /* @@ -362,8 +374,8 @@ pgstat_report_analyze(Relation rel, pgstat_unlock_entry(entry_ref); /* see pgstat_report_vacuum() */ - pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); + pgstat_flush_io(false, true); + (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO, true); } /* @@ -802,6 +814,29 @@ pgstat_twophase_postabort(FullTransactionId fxid, uint16 info, rec->tuples_inserted + rec->tuples_updated; } +/* + * Helper function to flush non-transactional statistics. + */ +static void +flush_relation_in_transaction_stats(PgStat_StatTabEntry *tabentry, PgStat_TableCounts *counts, + bool in_txn_only) +{ + TimestampTz t; + + tabentry->numscans += counts->numscans; + if (counts->numscans) + { + t = in_txn_only ? GetCurrentTimestamp() : GetCurrentTransactionStopTimestamp(); + if (t > tabentry->lastscan) + tabentry->lastscan = t; + } + + tabentry->tuples_returned += counts->tuples_returned; + tabentry->tuples_fetched += counts->tuples_fetched; + tabentry->blocks_fetched += counts->blocks_fetched; + tabentry->blocks_hit += counts->blocks_hit; +} + /* * Flush out pending stats for the entry * @@ -810,9 +845,17 @@ pgstat_twophase_postabort(FullTransactionId fxid, uint16 info, * * Some of the stats are copied to the corresponding pending database stats * entry when successfully flushing. + * + * If in_txn_only is true, only non-transactional fields are flushed + * (numscans, tuples_returned, tuples_fetched, blocks_fetched, blocks_hit). + * Transactional fields remain pending until transaction boundary. + * + * Some of the stats are copied to the corresponding pending database stats + * entry when successfully flushing. */ bool -pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial) { Oid dboid; PgStat_TableStatus *lstats; /* pending stats entry */ @@ -824,6 +867,9 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) lstats = (PgStat_TableStatus *) entry_ref->pending; shtabstats = (PgStatShared_Relation *) entry_ref->shared_stats; + /* this is a partial flush if in in_txn_only mode */ + *is_partial = in_txn_only; + /* * Ignore entries that didn't accumulate any actual counts, such as * indexes that were opened by the planner but not used. @@ -835,19 +881,36 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) if (!pgstat_lock_entry(entry_ref, nowait)) return false; - /* add the values to the shared entry. */ tabentry = &shtabstats->stats; - tabentry->numscans += lstats->counts.numscans; - if (lstats->counts.numscans) + if (in_txn_only) { - TimestampTz t = GetCurrentTransactionStopTimestamp(); - if (t > tabentry->lastscan) - tabentry->lastscan = t; + /* Flush non-transactional statistics */ + flush_relation_in_transaction_stats(tabentry, &lstats->counts, true); + + pgstat_unlock_entry(entry_ref); + + /* Also update the corresponding fields in database stats */ + dbentry = pgstat_prep_database_pending(dboid); + UPDATE_DATABASE_IN_TRANSACTION_STATS(dbentry, &lstats->counts); + + /* + * Clear the flushed fields from pending stats to prevent + * double-counting when we flush all fields at transaction boundary. + */ + lstats->counts.numscans = 0; + lstats->counts.tuples_returned = 0; + lstats->counts.tuples_fetched = 0; + lstats->counts.blocks_fetched = 0; + lstats->counts.blocks_hit = 0; + + return true; } - tabentry->tuples_returned += lstats->counts.tuples_returned; - tabentry->tuples_fetched += lstats->counts.tuples_fetched; + + /* Flush non-transactional statistics */ + flush_relation_in_transaction_stats(tabentry, &lstats->counts, false); + tabentry->tuples_inserted += lstats->counts.tuples_inserted; tabentry->tuples_updated += lstats->counts.tuples_updated; tabentry->tuples_deleted += lstats->counts.tuples_deleted; @@ -877,9 +940,6 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) */ tabentry->ins_since_vacuum += lstats->counts.tuples_inserted; - tabentry->blocks_fetched += lstats->counts.blocks_fetched; - tabentry->blocks_hit += lstats->counts.blocks_hit; - /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); /* Likewise for dead_tuples */ @@ -889,13 +949,11 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) /* The entry was successfully flushed, add the same to database stats */ dbentry = pgstat_prep_database_pending(dboid); - dbentry->tuples_returned += lstats->counts.tuples_returned; - dbentry->tuples_fetched += lstats->counts.tuples_fetched; + UPDATE_DATABASE_IN_TRANSACTION_STATS(dbentry, &lstats->counts); + dbentry->tuples_inserted += lstats->counts.tuples_inserted; dbentry->tuples_updated += lstats->counts.tuples_updated; dbentry->tuples_deleted += lstats->counts.tuples_deleted; - dbentry->blocks_fetched += lstats->counts.blocks_fetched; - dbentry->blocks_hit += lstats->counts.blocks_hit; return true; } diff --git a/src/backend/utils/activity/pgstat_slru.c b/src/backend/utils/activity/pgstat_slru.c index 2190f388eae..d8dc7be11ef 100644 --- a/src/backend/utils/activity/pgstat_slru.c +++ b/src/backend/utils/activity/pgstat_slru.c @@ -139,7 +139,7 @@ pgstat_get_slru_index(const char *name) * acquired. Otherwise return false. */ bool -pgstat_slru_flush_cb(bool nowait) +pgstat_slru_flush_cb(bool nowait, bool in_txn_only) { PgStatShared_SLRU *stats_shmem = &pgStatLocal.shmem->slru; int i; diff --git a/src/backend/utils/activity/pgstat_subscription.c b/src/backend/utils/activity/pgstat_subscription.c index 3277cf88a4e..4f289683d33 100644 --- a/src/backend/utils/activity/pgstat_subscription.c +++ b/src/backend/utils/activity/pgstat_subscription.c @@ -117,14 +117,20 @@ pgstat_fetch_stat_subscription(Oid subid) * false without flushing the entry. Otherwise returns true. */ bool -pgstat_subscription_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +pgstat_subscription_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial) { PgStat_BackendSubEntry *localent; PgStatShared_Subscription *shsubent; + Assert(!in_txn_only); + localent = (PgStat_BackendSubEntry *) entry_ref->pending; shsubent = (PgStatShared_Subscription *) entry_ref->shared_stats; + /* this is not a partial flush */ + *is_partial = false; + /* localent always has non-zero content */ if (!pgstat_lock_entry(entry_ref, nowait)) diff --git a/src/backend/utils/activity/pgstat_wal.c b/src/backend/utils/activity/pgstat_wal.c index 183e0a7a97b..bf796ad655e 100644 --- a/src/backend/utils/activity/pgstat_wal.c +++ b/src/backend/utils/activity/pgstat_wal.c @@ -51,12 +51,12 @@ pgstat_report_wal(bool force) nowait = !force; /* flush wal stats */ - (void) pgstat_wal_flush_cb(nowait); - pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_WAL); + (void) pgstat_wal_flush_cb(nowait, true); + (void) pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_WAL, true); /* flush IO stats */ - pgstat_flush_io(nowait); - (void) pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_IO); + pgstat_flush_io(nowait, true); + (void) pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_IO, true); } /* @@ -88,7 +88,7 @@ pgstat_wal_have_pending(void) * acquired. Otherwise return false. */ bool -pgstat_wal_flush_cb(bool nowait) +pgstat_wal_flush_cb(bool nowait, bool in_txn_only) { PgStatShared_Wal *stats_shmem = &pgStatLocal.shmem->wal; WalUsage wal_usage_diff = {0}; diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index 36ad708b360..bcce54367d5 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -40,6 +40,7 @@ volatile sig_atomic_t IdleSessionTimeoutPending = false; volatile sig_atomic_t ProcSignalBarrierPending = false; volatile sig_atomic_t LogMemoryContextPending = false; volatile sig_atomic_t IdleStatsUpdateTimeoutPending = false; +volatile sig_atomic_t InTransactionStatsUpdateTimeoutPending = false; volatile uint32 InterruptHoldoffCount = 0; volatile uint32 QueryCancelHoldoffCount = 0; volatile uint32 CritSectionCount = 0; diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 26118661f07..27ecf0df16e 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -65,6 +65,7 @@ #include "utils/injection_point.h" #include "utils/memutils.h" #include "utils/pg_locale.h" +#include "utils/pgstat_internal.h" #include "utils/portal.h" #include "utils/ps_status.h" #include "utils/snapmgr.h" @@ -89,6 +90,7 @@ static void IdleInTransactionSessionTimeoutHandler(void); static void TransactionTimeoutHandler(void); static void IdleSessionTimeoutHandler(void); static void IdleStatsUpdateTimeoutHandler(void); +static void InTransactionStatsUpdateTimeoutHandler(void); static void ClientCheckTimeoutHandler(void); static bool ThereIsAtLeastOneRole(void); static void process_startup_options(Port *port, bool am_superuser); @@ -774,6 +776,8 @@ InitPostgres(const char *in_dbname, Oid dboid, RegisterTimeout(CLIENT_CONNECTION_CHECK_TIMEOUT, ClientCheckTimeoutHandler); RegisterTimeout(IDLE_STATS_UPDATE_TIMEOUT, IdleStatsUpdateTimeoutHandler); + RegisterTimeout(IN_TRANSACTION_STATS_UPDATE_TIMEOUT, + InTransactionStatsUpdateTimeoutHandler); } /* @@ -1433,6 +1437,14 @@ IdleStatsUpdateTimeoutHandler(void) SetLatch(MyLatch); } +static void +InTransactionStatsUpdateTimeoutHandler(void) +{ + InTransactionStatsUpdateTimeoutPending = true; + InterruptPending = true; + SetLatch(MyLatch); +} + static void ClientCheckTimeoutHandler(void) { diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index f16f35659b9..7051e4b6f3f 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -96,6 +96,7 @@ extern PGDLLIMPORT volatile sig_atomic_t IdleSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t ProcSignalBarrierPending; extern PGDLLIMPORT volatile sig_atomic_t LogMemoryContextPending; extern PGDLLIMPORT volatile sig_atomic_t IdleStatsUpdateTimeoutPending; +extern PGDLLIMPORT volatile sig_atomic_t InTransactionStatsUpdateTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t CheckClientConnectionPending; extern PGDLLIMPORT volatile sig_atomic_t ClientConnectionLost; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 216b93492ba..8571eb1afd7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -38,6 +38,13 @@ typedef struct RelationData *Relation; /* Default directory to store temporary statistics data in */ #define PG_STAT_TMP_DIR "pg_stat_tmp" +/* + * Interval in milliseconds for flushing FLUSH_IN_TRANSACTION stats to shared + * memory. An explicit transaction always enters idle-in-transaction state + * between commands, which is when this timeout is enabled. + */ +#define PGSTAT_IDLE_TXN_INTERVAL 10000 + /* Values for track_functions GUC variable --- order is significant! */ typedef enum TrackFunctionsLevel { @@ -536,6 +543,7 @@ extern void pgstat_initialize(void); /* Functions called from backends */ extern long pgstat_report_stat(bool force); +extern void pgstat_report_in_transaction_stat(bool force); extern void pgstat_force_next_flush(void); extern void pgstat_reset_counters(void); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 9b8fbae00ed..e9f4dc85528 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -224,6 +224,18 @@ typedef struct PgStat_SubXactStatus PgStat_TableXactStatus *first; /* head of list for this subxact */ } PgStat_SubXactStatus; +/* + * Flush mode for statistics kinds. + * + * FLUSH_AT_TXN_BOUNDARY has to be the first because we want it to be the + * default value. + */ +typedef enum PgStat_FlushMode +{ + FLUSH_AT_TXN_BOUNDARY, /* All fields can only be flushed at + * transaction boundary */ + FLUSH_IN_TRANSACTION, /* All fields can be flushed in a transaction */ +} PgStat_FlushMode; /* * Metadata for a specific kind of statistics. @@ -251,6 +263,16 @@ typedef struct PgStat_KindInfo */ bool track_entry_count:1; + /* + * The mode of when to flush stats. See PgStat_FlushMode for more details. + * + * This member only has meaning for statistics kinds that accumulate + * pending stats and use flush callbacks. For kinds that write directly to + * shared memory (e.g., archiver, bgwriter, checkpointer), this member has + * no effect. + */ + PgStat_FlushMode flush_mode; + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is @@ -297,8 +319,13 @@ typedef struct PgStat_KindInfo * For variable-numbered stats: flush pending stats. Required if pending * data is used. See flush_static_cb when dealing with stats data that * that cannot use PgStat_EntryRef->pending. + * + * The in_txn_only parameter indicates whether this is an in-transaction + * flush. The is_partial parameter indicates whether this is a partial + * flush. */ - bool (*flush_pending_cb) (PgStat_EntryRef *sr, bool nowait); + bool (*flush_pending_cb) (PgStat_EntryRef *sr, bool nowait, + bool in_txn_only, bool *is_partial); /* * For variable-numbered stats: delete pending stats. Optional. @@ -366,8 +393,11 @@ typedef struct PgStat_KindInfo * * "pgstat_report_fixed" needs to be set to trigger the flush of pending * stats. + * + * The in_txn_only parameter indicates whether this is an in-transaction + * flush. */ - bool (*flush_static_cb) (bool nowait); + bool (*flush_static_cb) (bool nowait, bool in_txn_only); /* * For fixed-numbered statistics: Reset All. @@ -696,8 +726,8 @@ extern void pgstat_archiver_snapshot_cb(void); #define PGSTAT_BACKEND_FLUSH_WAL (1 << 1) /* Flush WAL statistics */ #define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL) -extern bool pgstat_flush_backend(bool nowait, bits32 flags); -extern bool pgstat_backend_flush_cb(bool nowait); +extern bool pgstat_flush_backend(bool nowait, bits32 flags, bool in_txn_only); +extern bool pgstat_backend_flush_cb(bool nowait, bool in_txn_only); extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); @@ -729,7 +759,8 @@ extern void AtEOXact_PgStat_Database(bool isCommit, bool parallel); extern PgStat_StatDBEntry *pgstat_prep_database_pending(Oid dboid); extern void pgstat_reset_database_timestamp(Oid dboid, TimestampTz ts); -extern bool pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial); extern void pgstat_database_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); @@ -737,7 +768,8 @@ extern void pgstat_database_reset_timestamp_cb(PgStatShared_Common *header, Time * Functions in pgstat_function.c */ -extern bool pgstat_function_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_function_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial); extern void pgstat_function_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); @@ -745,9 +777,9 @@ extern void pgstat_function_reset_timestamp_cb(PgStatShared_Common *header, Time * Functions in pgstat_io.c */ -extern void pgstat_flush_io(bool nowait); +extern void pgstat_flush_io(bool nowait, bool in_txn_only); -extern bool pgstat_io_flush_cb(bool nowait); +extern bool pgstat_io_flush_cb(bool nowait, bool in_txn_only); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); @@ -762,7 +794,8 @@ extern void AtEOSubXact_PgStat_Relations(PgStat_SubXactStatus *xact_state, bool extern void AtPrepare_PgStat_Relations(PgStat_SubXactStatus *xact_state); extern void PostPrepare_PgStat_Relations(PgStat_SubXactStatus *xact_state); -extern bool pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial); extern void pgstat_relation_delete_pending_cb(PgStat_EntryRef *entry_ref); extern void pgstat_relation_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); @@ -809,7 +842,7 @@ extern PgStatShared_Common *pgstat_init_entry(PgStat_Kind kind, * Functions in pgstat_slru.c */ -extern bool pgstat_slru_flush_cb(bool nowait); +extern bool pgstat_slru_flush_cb(bool nowait, bool in_txn_only); extern void pgstat_slru_init_shmem_cb(void *stats); extern void pgstat_slru_reset_all_cb(TimestampTz ts); extern void pgstat_slru_snapshot_cb(void); @@ -820,7 +853,7 @@ extern void pgstat_slru_snapshot_cb(void); */ extern void pgstat_wal_init_backend_cb(void); -extern bool pgstat_wal_flush_cb(bool nowait); +extern bool pgstat_wal_flush_cb(bool nowait, bool in_txn_only); extern void pgstat_wal_init_shmem_cb(void *stats); extern void pgstat_wal_reset_all_cb(TimestampTz ts); extern void pgstat_wal_snapshot_cb(void); @@ -830,7 +863,8 @@ extern void pgstat_wal_snapshot_cb(void); * Functions in pgstat_subscription.c */ -extern bool pgstat_subscription_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_subscription_flush_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial); extern void pgstat_subscription_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); diff --git a/src/include/utils/timeout.h b/src/include/utils/timeout.h index 0965b590b34..68fe814bde6 100644 --- a/src/include/utils/timeout.h +++ b/src/include/utils/timeout.h @@ -34,6 +34,7 @@ typedef enum TimeoutId TRANSACTION_TIMEOUT, IDLE_SESSION_TIMEOUT, IDLE_STATS_UPDATE_TIMEOUT, + IN_TRANSACTION_STATS_UPDATE_TIMEOUT, CLIENT_CONNECTION_CHECK_TIMEOUT, STARTUP_PROGRESS_TIMEOUT, /* First user-definable timeout reason */ diff --git a/src/test/modules/test_custom_stats/t/001_custom_stats.pl b/src/test/modules/test_custom_stats/t/001_custom_stats.pl index 9e6a7a38577..d3afbff37c5 100644 --- a/src/test/modules/test_custom_stats/t/001_custom_stats.pl +++ b/src/test/modules/test_custom_stats/t/001_custom_stats.pl @@ -156,5 +156,64 @@ $result = $node->safe_psql('postgres', ); is($result, "0", "report of fixed-sized after manual reset"); +# Test FLUSH_IN_TRANSACTION custom stats kind. +# The intxn kind is registered alongside the regular kind by test_custom_var_stats. + +# Basic update and report. +$node->safe_psql('postgres', + q(SELECT test_custom_var_intxn_update('intxn_entry1'))); +$node->safe_psql('postgres', + q(SELECT test_custom_var_intxn_update('intxn_entry1'))); +$node->safe_psql('postgres', + q(SELECT test_custom_var_intxn_update('intxn_entry1'))); + +$result = $node->safe_psql('postgres', + q(SELECT test_custom_var_intxn_report('intxn_entry1'))); +is($result, "3", "intxn stats report after updates"); + +# Verify intxn stats are not persisted (write_to_file = false). +$node->stop(); +$node->start(); + +$result = $node->safe_psql('postgres', + q(SELECT test_custom_var_intxn_report('intxn_entry1'))); +is($result, "", "intxn stats lost after clean restart (not persisted)"); + +# Test in-transaction flushing with injection point. +SKIP: +{ + skip "injection points not supported", 1 + unless (($ENV{enable_injection_points} // '') eq 'yes' + && $node->check_extension('injection_points')); + + $node->safe_psql('postgres', 'CREATE EXTENSION IF NOT EXISTS injection_points;'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('in-transaction-stats-short-interval', 'error');"); + + $node->append_conf('postgresql.conf', 'stats_fetch_consistency = none'); + $node->reload; + + my $intxn_before = $node->safe_psql('postgres', + "SELECT COALESCE(test_custom_var_intxn_report('intxn_flush'), 0);"); + + $result = $node->safe_psql('postgres', q{ +BEGIN; +SELECT test_custom_var_intxn_update('intxn_flush'); +SELECT test_custom_var_intxn_update('intxn_flush'); +SELECT test_custom_var_intxn_update('intxn_flush'); +SELECT pg_sleep(2); +SELECT COALESCE(test_custom_var_intxn_report('intxn_flush'), 0); +}); + + my @lines = split(/\n/, $result); + my $intxn_mid_txn = $lines[-1]; + + ok($intxn_mid_txn > $intxn_before, + "custom intxn stats flushed mid-transaction (before: $intxn_before, mid-txn: $intxn_mid_txn)"); + + $node->safe_psql('postgres', + "SELECT injection_points_detach('in-transaction-stats-short-interval');"); +} + # Test completed successfully done_testing(); diff --git a/src/test/modules/test_custom_stats/test_custom_var_stats--1.0.sql b/src/test/modules/test_custom_stats/test_custom_var_stats--1.0.sql index 5ed8cfc2dcf..03650cbc414 100644 --- a/src/test/modules/test_custom_stats/test_custom_var_stats--1.0.sql +++ b/src/test/modules/test_custom_stats/test_custom_var_stats--1.0.sql @@ -24,3 +24,13 @@ CREATE FUNCTION test_custom_stats_var_report(INOUT name TEXT, RETURNS SETOF record AS 'MODULE_PATHNAME', 'test_custom_stats_var_report' LANGUAGE C STRICT PARALLEL UNSAFE; + +CREATE FUNCTION test_custom_var_intxn_update(IN name TEXT) +RETURNS void +AS 'MODULE_PATHNAME', 'test_custom_var_intxn_update' +LANGUAGE C STRICT PARALLEL UNSAFE; + +CREATE FUNCTION test_custom_var_intxn_report(IN name TEXT) +RETURNS bigint +AS 'MODULE_PATHNAME', 'test_custom_var_intxn_report' +LANGUAGE C STRICT PARALLEL UNSAFE; diff --git a/src/test/modules/test_custom_stats/test_custom_var_stats.c b/src/test/modules/test_custom_stats/test_custom_var_stats.c index 2ef0e903745..e87d47a28ab 100644 --- a/src/test/modules/test_custom_stats/test_custom_var_stats.c +++ b/src/test/modules/test_custom_stats/test_custom_var_stats.c @@ -37,6 +37,11 @@ PG_MODULE_MAGIC_EXT( */ #define PGSTAT_KIND_TEST_CUSTOM_VAR_STATS 25 +/* + * Kind ID for test_custom_var_intxn_stats statistics (FLUSH_IN_TRANSACTION). + */ +#define PGSTAT_KIND_TEST_CUSTOM_VAR_INTXN_STATS 27 + /* File paths for auxiliary data serialization */ #define TEST_CUSTOM_AUX_DATA_DESC "pg_stat/test_custom_var_stats_desc.stats" @@ -45,6 +50,8 @@ PG_MODULE_MAGIC_EXT( */ #define PGSTAT_CUSTOM_VAR_STATS_IDX(name) hash_bytes_extended((const unsigned char *) name, strlen(name), 0) +#define PGSTAT_CUSTOM_VAR_INTXN_IDX(name) hash_bytes_extended((const unsigned char *) name, strlen(name), 0) + /*-------------------------------------------------------------------------- * Type definitions *-------------------------------------------------------------------------- @@ -56,6 +63,12 @@ typedef struct PgStat_StatCustomVarEntry PgStat_Counter numcalls; /* times statistic was incremented */ } PgStat_StatCustomVarEntry; +/* Pending entry for FLUSH_IN_TRANSACTION kind (same layout) */ +typedef struct PgStat_CustomVarInTxnEntry +{ + PgStat_Counter numcalls; +} PgStat_CustomVarInTxnEntry; + /* Shared memory statistics entry visible to all backends */ typedef struct PgStatShared_CustomVarEntry { @@ -64,6 +77,13 @@ typedef struct PgStatShared_CustomVarEntry dsa_pointer description; /* pointer to description string in DSA */ } PgStatShared_CustomVarEntry; +/* Shared memory entry for FLUSH_IN_TRANSACTION kind */ +typedef struct PgStatShared_CustomVarInTxnEntry +{ + PgStatShared_Common header; + PgStat_CustomVarInTxnEntry stats; +} PgStatShared_CustomVarInTxnEntry; + /*-------------------------------------------------------------------------- * Global Variables *-------------------------------------------------------------------------- @@ -85,7 +105,8 @@ static dsa_area *custom_stats_description_dsa = NULL; /* Flush callback: merge pending stats into shared memory */ static bool test_custom_stats_var_flush_pending_cb(PgStat_EntryRef *entry_ref, - bool nowait); + bool nowait, bool in_txn_only, + bool *is_partial); /* Serialization callback: write auxiliary entry data */ static void test_custom_stats_var_to_serialized_data(const PgStat_HashKey *key, @@ -100,6 +121,11 @@ static bool test_custom_stats_var_from_serialized_data(const PgStat_HashKey *key /* Finish callback: end of statistics file operations */ static void test_custom_stats_var_finish(PgStat_StatsFileOp status); +/* Flush callback for FLUSH_IN_TRANSACTION kind */ +static bool test_custom_var_intxn_flush_pending_cb(PgStat_EntryRef *entry_ref, + bool nowait, bool in_txn_only, + bool *is_partial); + /*-------------------------------------------------------------------------- * Custom kind configuration *-------------------------------------------------------------------------- @@ -121,6 +147,19 @@ static const PgStat_KindInfo custom_stats = { .finish = test_custom_stats_var_finish, }; +static const PgStat_KindInfo custom_intxn_stats = { + .name = "test_custom_var_intxn_stats", + .fixed_amount = false, + .write_to_file = false, /* no persistence needed for test */ + .flush_mode = FLUSH_IN_TRANSACTION, + .accessed_across_databases = true, + .shared_size = sizeof(PgStatShared_CustomVarInTxnEntry), + .shared_data_off = offsetof(PgStatShared_CustomVarInTxnEntry, stats), + .shared_data_len = sizeof(PgStat_CustomVarInTxnEntry), + .pending_size = sizeof(PgStat_CustomVarInTxnEntry), + .flush_pending_cb = test_custom_var_intxn_flush_pending_cb, +}; + /*-------------------------------------------------------------------------- * Module initialization *-------------------------------------------------------------------------- @@ -133,8 +172,9 @@ _PG_init(void) if (!process_shared_preload_libraries_in_progress) return; - /* Register custom statistics kind */ + /* Register custom statistics kinds */ pgstat_register_kind(PGSTAT_KIND_TEST_CUSTOM_VAR_STATS, &custom_stats); + pgstat_register_kind(PGSTAT_KIND_TEST_CUSTOM_VAR_INTXN_STATS, &custom_intxn_stats); } /*-------------------------------------------------------------------------- @@ -152,7 +192,8 @@ _PG_init(void) * Returns false only if nowait=true and lock acquisition fails. */ static bool -test_custom_stats_var_flush_pending_cb(PgStat_EntryRef *entry_ref, bool nowait) +test_custom_stats_var_flush_pending_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial) { PgStat_StatCustomVarEntry *pending_entry; PgStatShared_CustomVarEntry *shared_entry; @@ -160,6 +201,9 @@ test_custom_stats_var_flush_pending_cb(PgStat_EntryRef *entry_ref, bool nowait) pending_entry = (PgStat_StatCustomVarEntry *) entry_ref->pending; shared_entry = (PgStatShared_CustomVarEntry *) entry_ref->shared_stats; + /* this is not a partial flush */ + *is_partial = false; + if (!pgstat_lock_entry(entry_ref, nowait)) return false; @@ -691,3 +735,67 @@ test_custom_stats_var_report(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } + +/*-------------------------------------------------------------------------- + * FLUSH_IN_TRANSACTION kind: callbacks and SQL functions + *-------------------------------------------------------------------------- + */ + +static bool +test_custom_var_intxn_flush_pending_cb(PgStat_EntryRef *entry_ref, bool nowait, + bool in_txn_only, bool *is_partial) +{ + PgStat_CustomVarInTxnEntry *pending; + PgStatShared_CustomVarInTxnEntry *shared; + + pending = (PgStat_CustomVarInTxnEntry *) entry_ref->pending; + shared = (PgStatShared_CustomVarInTxnEntry *) entry_ref->shared_stats; + + *is_partial = false; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + + shared->stats.numcalls += pending->numcalls; + + pgstat_unlock_entry(entry_ref); + + return true; +} + +PG_FUNCTION_INFO_V1(test_custom_var_intxn_update); +Datum +test_custom_var_intxn_update(PG_FUNCTION_ARGS) +{ + char *stat_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + PgStat_EntryRef *entry_ref; + PgStat_CustomVarInTxnEntry *pending; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_TEST_CUSTOM_VAR_INTXN_STATS, + InvalidOid, + PGSTAT_CUSTOM_VAR_INTXN_IDX(stat_name), + NULL); + + pending = (PgStat_CustomVarInTxnEntry *) entry_ref->pending; + pending->numcalls++; + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(test_custom_var_intxn_report); +Datum +test_custom_var_intxn_report(PG_FUNCTION_ARGS) +{ + char *stat_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + PgStat_CustomVarInTxnEntry *stats; + + stats = (PgStat_CustomVarInTxnEntry *) + pgstat_fetch_entry(PGSTAT_KIND_TEST_CUSTOM_VAR_INTXN_STATS, + InvalidOid, + PGSTAT_CUSTOM_VAR_INTXN_IDX(stat_name)); + + if (!stats) + PG_RETURN_NULL(); + + PG_RETURN_INT64(stats->numcalls); +} diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 6e8db1621a7..70248360d21 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -19,6 +19,7 @@ tests += { 't/008_replslot_single_user.pl', 't/009_log_temp_files.pl', 't/010_index_concurrently_upsert.pl', + 't/011_in_transaction_stats.pl', ], # The injection points are cluster-wide, so disable installcheck 'runningcheck': false, diff --git a/src/test/modules/test_misc/t/011_in_transaction_stats.pl b/src/test/modules/test_misc/t/011_in_transaction_stats.pl new file mode 100644 index 00000000000..cf1bde24a5d --- /dev/null +++ b/src/test/modules/test_misc/t/011_in_transaction_stats.pl @@ -0,0 +1,91 @@ + +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +# Test in-transaction stats flushing mechanism. +# +# This test verifies that FLUSH_IN_TRANSACTION stats are periodically flushed +# to shared memory while a transaction is idle between commands. Uses an +# injection point to reduce the flush interval from 10 seconds to 1 second +# for faster testing. +# +# We test one representative of each stats kind: relation stats (seq_scan) +# for variable-numbered stats, and WAL for fixed-sized stats. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init(); +$node->append_conf('postgresql.conf', 'stats_fetch_consistency = none'); +$node->start; + +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +# Attach injection point to reduce the in-transaction stats flush interval +# to 1 second. +$node->safe_psql('postgres', + "SELECT injection_points_attach('in-transaction-stats-short-interval', 'error');"); + +# Create test table +$node->safe_psql('postgres', + 'CREATE TABLE test_in_txn_stats(a int) WITH (autovacuum_enabled = off);'); +$node->safe_psql('postgres', + 'INSERT INTO test_in_txn_stats SELECT generate_series(1, 1000);'); + +# Force flush and get baseline stats +$node->safe_psql('postgres', 'SELECT pg_stat_force_next_flush();'); +my $seq_scan_before = $node->safe_psql('postgres', + "SELECT seq_scan FROM pg_stat_user_tables WHERE relname = 'test_in_txn_stats';"); + +# Test that seq_scan stats (variable-numbered) are flushed mid-transaction +# via the in-transaction periodic flush mechanism. +my $result = $node->safe_psql('postgres', q{ +BEGIN; +SELECT COUNT(*) FROM test_in_txn_stats; +SELECT COUNT(*) FROM test_in_txn_stats; +SELECT pg_sleep(2); +SELECT seq_scan FROM pg_stat_user_tables WHERE relname = 'test_in_txn_stats'; +}); + +my @lines = split(/\n/, $result); +my $seq_scan_mid_txn = $lines[-1]; + +ok($seq_scan_mid_txn > $seq_scan_before, + "seq_scan stats flushed during transaction (before: $seq_scan_before, mid-txn: $seq_scan_mid_txn)"); + +# Test WAL stats (fixed-sized) flushing during transaction. +$node->safe_psql('postgres', 'SELECT pg_stat_reset_shared(\'wal\');'); +my $wal_records_before = $node->safe_psql('postgres', + "SELECT wal_records FROM pg_stat_wal;"); + +$result = $node->safe_psql('postgres', q{ +BEGIN; +INSERT INTO test_in_txn_stats SELECT generate_series(1, 1000); +SELECT pg_sleep(2); +SELECT wal_records FROM pg_stat_wal; +}); + +@lines = split(/\n/, $result); +my $wal_records_mid_txn = $lines[-1]; + +ok($wal_records_mid_txn > $wal_records_before, + "WAL stats flushed during transaction (before: $wal_records_before, mid-txn: $wal_records_mid_txn)"); + +# Cleanup +$node->safe_psql('postgres', 'DROP TABLE test_in_txn_stats;'); + +done_testing(); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 52f8603a7be..c9cd6087abc 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2290,6 +2290,7 @@ PgStat_Counter PgStat_EntryRef PgStat_EntryRefHashEntry PgStat_FetchConsistency +PgStat_FlushMode PgStat_FunctionCallUsage PgStat_FunctionCounts PgStat_HashKey -- 2.47.3 ^ permalink raw reply [nested|flat] 58+ messages in thread
* Re: Flush some statistics within running transactions @ 2026-03-25 03:16 Michael Paquier <[email protected]> parent: Sami Imseih <[email protected]> 0 siblings, 0 replies; 58+ messages in thread From: Michael Paquier @ 2026-03-25 03:16 UTC (permalink / raw) To: Sami Imseih <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; Fujii Masao <[email protected]>; [email protected]; Zsolt Parragi <[email protected]> On Mon, Mar 16, 2026 at 06:42:44PM -0500, Sami Imseih wrote: > So attached is a new proposal with tests and docs. In terms of > test I fell back to the strategy used by Bertrand [0] with the As far as I am reading the patch, it seems to me that this would accelerate the frequency of the stats flushes when we have a transaction with many short queries, but it does not help much in the case of an analytical single query because the flush of the stats would just happen once after the timeout set for each query? Let's imagine for example a query that takes hours to run, where we'd want to get fresh IO and WAL stats (backend included) on a periodic basis by processing interrupts while going through the executor. Thinking more about this problem, I'd really like to think that a client-side API would provide a more flexible interface for the analytical case. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 58+ messages in thread
end of thread, other threads:[~2026-03-25 03:16 UTC | newest] Thread overview: 58+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v13 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v16 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v10 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v9 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v19 3/5] Remove globals readOff, readLen and readSegNo. Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v8 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v15 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2019-09-10 08:28 [PATCH v17 3/4] Remove globals readOff, readLen and readSegNo Kyotaro Horiguchi <[email protected]> 2021-09-30 04:16 [PATCH v18 3/5] Remove globals readOff, readLen and readSegNo. Kyotaro Horiguchi <[email protected]> 2025-02-19 17:08 Re: why there is not VACUUM FULL CONCURRENTLY? Antonin Houska <[email protected]> 2025-02-19 17:40 ` Re: why there is not VACUUM FULL CONCURRENTLY? Marcos Pegoraro <[email protected]> 2026-03-16 06:26 Re: Flush some statistics within running transactions Michael Paquier <[email protected]> 2026-03-16 09:20 ` Re: Flush some statistics within running transactions Bertrand Drouvot <[email protected]> 2026-03-16 10:22 ` Re: Flush some statistics within running transactions Michael Paquier <[email protected]> 2026-03-16 23:42 ` Re: Flush some statistics within running transactions Sami Imseih <[email protected]> 2026-03-25 03:16 ` Re: Flush some statistics within running transactions Michael Paquier <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox