public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 7/7] Move code to apply one WAL record to a subroutine. 21+ messages / 5 participants [nested] [flat]
* [PATCH 7/7] Move code to apply one WAL record to a subroutine. @ 2021-06-21 21:00 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-06-21 21:00 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 283 +++++++++++----------- 1 file changed, 148 insertions(+), 135 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 5e23244f6da..c78fc5273bd 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -366,6 +366,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1398,11 +1399,8 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; - XLogRecPtr ReadRecPtr; - XLogRecPtr EndRecPtr; pg_rusage_init(&ru0); @@ -1424,11 +1422,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - - ReadRecPtr = xlogreader->ReadRecPtr; - EndRecPtr = xlogreader->EndRecPtr; - #ifdef WAL_DEBUG if (XLOG_DEBUG || (rmid == RM_XACT_ID && trace_recovery_messages <= DEBUG2) || @@ -1438,8 +1431,8 @@ PerformWalRecovery(void) initStringInfo(&buf); appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ", - LSN_FORMAT_ARGS(ReadRecPtr), - LSN_FORMAT_ARGS(EndRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->EndRecPtr)); xlog_outrec(&buf, xlogreader); appendStringInfoString(&buf, " - "); xlog_outdesc(&buf, xlogreader); @@ -1494,132 +1487,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - /* - * ShmemVariableCache->nextXid must be beyond record's xid. + * Apply the record */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newTLI = ThisTimeLineID; - TimeLineID prevTLI = ThisTimeLineID; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newTLI = checkPoint.ThisTimeLineID; - prevTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newTLI = xlrec.ThisTimeLineID; - prevTLI = xlrec.PrevTimeLineID; - } - - if (newTLI != ThisTimeLineID) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); - - /* Following WAL records should be run with new TLI */ - ThisTimeLineID = newTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->replayEndRecPtr = EndRecPtr; - XLogRecCtl->replayEndTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; - XLogRecCtl->lastReplayedTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = ReadRecPtr; - - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. - */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1678,7 +1549,7 @@ PerformWalRecovery(void) ereport(LOG, (errmsg("redo done at %X/%X system usage: %s", - LSN_FORMAT_ARGS(ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) @@ -1707,6 +1578,148 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record) +{ + XLogRecPtr ReadRecPtr; + XLogRecPtr EndRecPtr; + ErrorContextCallback errcallback; + bool switchedTLI = false; + + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the + * current timeline to change. The record is already considered to + * be part of the new timeline, so we update ThisTimeLineID before + * replaying it. That's important so that replayEndTLI, which is + * recorded as the minimum recovery point's TLI if recovery stops + * after this record, is set correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newTLI = ThisTimeLineID; + TimeLineID prevTLI = ThisTimeLineID; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newTLI = checkPoint.ThisTimeLineID; + prevTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newTLI = xlrec.ThisTimeLineID; + prevTLI = xlrec.PrevTimeLineID; + } + + if (newTLI != ThisTimeLineID) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); + + /* Following WAL records should be run with new TLI */ + ThisTimeLineID = newTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so + * that XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->replayEndRecPtr = EndRecPtr; + XLogRecCtl->replayEndTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we + * see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the + * WAL record are consistent with the existing pages. This check + * is done only if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been + * successfully replayed. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; + XLogRecCtl->lastReplayedTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up + * the receiver so that it notices the updated + * lastReplayedEndRecPtr and sends a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any + * (possibly bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); + + /* + * Wake up any walsenders to notice that we are on a new + * timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------8C95DA69A2E7D2FC9457CD47-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 7/7] Move code to apply one WAL record to a subroutine. @ 2021-06-21 21:00 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-06-21 21:00 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 283 +++++++++++----------- 1 file changed, 148 insertions(+), 135 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 44eb425eaf9..d7787c9a082 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -366,6 +366,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1392,11 +1393,8 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; - XLogRecPtr ReadRecPtr; - XLogRecPtr EndRecPtr; pg_rusage_init(&ru0); @@ -1418,11 +1416,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - - ReadRecPtr = xlogreader->ReadRecPtr; - EndRecPtr = xlogreader->EndRecPtr; - #ifdef WAL_DEBUG if (XLOG_DEBUG || (rmid == RM_XACT_ID && trace_recovery_messages <= DEBUG2) || @@ -1432,8 +1425,8 @@ PerformWalRecovery(void) initStringInfo(&buf); appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ", - LSN_FORMAT_ARGS(ReadRecPtr), - LSN_FORMAT_ARGS(EndRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->EndRecPtr)); xlog_outrec(&buf, xlogreader); appendStringInfoString(&buf, " - "); xlog_outdesc(&buf, xlogreader); @@ -1488,132 +1481,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - /* - * ShmemVariableCache->nextXid must be beyond record's xid. + * Apply the record */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newTLI = ThisTimeLineID; - TimeLineID prevTLI = ThisTimeLineID; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newTLI = checkPoint.ThisTimeLineID; - prevTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newTLI = xlrec.ThisTimeLineID; - prevTLI = xlrec.PrevTimeLineID; - } - - if (newTLI != ThisTimeLineID) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); - - /* Following WAL records should be run with new TLI */ - ThisTimeLineID = newTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->replayEndRecPtr = EndRecPtr; - XLogRecCtl->replayEndTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; - XLogRecCtl->lastReplayedTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = ReadRecPtr; - - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. - */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1672,7 +1543,7 @@ PerformWalRecovery(void) ereport(LOG, (errmsg("redo done at %X/%X system usage: %s", - LSN_FORMAT_ARGS(ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) @@ -1701,6 +1572,148 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record) +{ + XLogRecPtr ReadRecPtr; + XLogRecPtr EndRecPtr; + ErrorContextCallback errcallback; + bool switchedTLI = false; + + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the + * current timeline to change. The record is already considered to + * be part of the new timeline, so we update ThisTimeLineID before + * replaying it. That's important so that replayEndTLI, which is + * recorded as the minimum recovery point's TLI if recovery stops + * after this record, is set correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newTLI = ThisTimeLineID; + TimeLineID prevTLI = ThisTimeLineID; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newTLI = checkPoint.ThisTimeLineID; + prevTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newTLI = xlrec.ThisTimeLineID; + prevTLI = xlrec.PrevTimeLineID; + } + + if (newTLI != ThisTimeLineID) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); + + /* Following WAL records should be run with new TLI */ + ThisTimeLineID = newTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so + * that XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->replayEndRecPtr = EndRecPtr; + XLogRecCtl->replayEndTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we + * see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the + * WAL record are consistent with the existing pages. This check + * is done only if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been + * successfully replayed. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; + XLogRecCtl->lastReplayedTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up + * the receiver so that it notices the updated + * lastReplayedEndRecPtr and sends a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any + * (possibly bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); + + /* + * Wake up any walsenders to notice that we are on a new + * timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------4DE063CB5604E65545208F19-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v5 3/3] Move code to apply one WAL record to a subroutine. @ 2021-07-31 12:06 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-07-31 12:06 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 283 +++++++++++----------- 1 file changed, 148 insertions(+), 135 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 6030d6fe819..85909c9b686 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -366,6 +366,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1374,11 +1375,8 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; - XLogRecPtr ReadRecPtr; - XLogRecPtr EndRecPtr; pg_rusage_init(&ru0); @@ -1400,11 +1398,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - - ReadRecPtr = xlogreader->ReadRecPtr; - EndRecPtr = xlogreader->EndRecPtr; - #ifdef WAL_DEBUG if (XLOG_DEBUG || (rmid == RM_XACT_ID && trace_recovery_messages <= DEBUG2) || @@ -1414,8 +1407,8 @@ PerformWalRecovery(void) initStringInfo(&buf); appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ", - LSN_FORMAT_ARGS(ReadRecPtr), - LSN_FORMAT_ARGS(EndRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->EndRecPtr)); xlog_outrec(&buf, xlogreader); appendStringInfoString(&buf, " - "); xlog_outdesc(&buf, xlogreader); @@ -1470,132 +1463,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - /* - * ShmemVariableCache->nextXid must be beyond record's xid. + * Apply the record */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newTLI = ThisTimeLineID; - TimeLineID prevTLI = ThisTimeLineID; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newTLI = checkPoint.ThisTimeLineID; - prevTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newTLI = xlrec.ThisTimeLineID; - prevTLI = xlrec.PrevTimeLineID; - } - - if (newTLI != ThisTimeLineID) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); - - /* Following WAL records should be run with new TLI */ - ThisTimeLineID = newTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->replayEndRecPtr = EndRecPtr; - XLogRecCtl->replayEndTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; - XLogRecCtl->lastReplayedTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = ReadRecPtr; - - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. - */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1654,7 +1525,7 @@ PerformWalRecovery(void) ereport(LOG, (errmsg("redo done at %X/%X system usage: %s", - LSN_FORMAT_ARGS(ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) @@ -1683,6 +1554,148 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record) +{ + XLogRecPtr ReadRecPtr; + XLogRecPtr EndRecPtr; + ErrorContextCallback errcallback; + bool switchedTLI = false; + + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the + * current timeline to change. The record is already considered to + * be part of the new timeline, so we update ThisTimeLineID before + * replaying it. That's important so that replayEndTLI, which is + * recorded as the minimum recovery point's TLI if recovery stops + * after this record, is set correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newTLI = ThisTimeLineID; + TimeLineID prevTLI = ThisTimeLineID; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newTLI = checkPoint.ThisTimeLineID; + prevTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newTLI = xlrec.ThisTimeLineID; + prevTLI = xlrec.PrevTimeLineID; + } + + if (newTLI != ThisTimeLineID) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); + + /* Following WAL records should be run with new TLI */ + ThisTimeLineID = newTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so + * that XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->replayEndRecPtr = EndRecPtr; + XLogRecCtl->replayEndTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we + * see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the + * WAL record are consistent with the existing pages. This check + * is done only if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been + * successfully replayed. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; + XLogRecCtl->lastReplayedTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up + * the receiver so that it notices the updated + * lastReplayedEndRecPtr and sends a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any + * (possibly bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); + + /* + * Wake up any walsenders to notice that we are on a new + * timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------F589ECF1004AF34308A0B206-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4 3/3] Move code to apply one WAL record to a subroutine. @ 2021-07-31 12:06 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-07-31 12:06 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 283 +++++++++++----------- 1 file changed, 148 insertions(+), 135 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 6030d6fe819..85909c9b686 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -366,6 +366,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1374,11 +1375,8 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; - XLogRecPtr ReadRecPtr; - XLogRecPtr EndRecPtr; pg_rusage_init(&ru0); @@ -1400,11 +1398,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - - ReadRecPtr = xlogreader->ReadRecPtr; - EndRecPtr = xlogreader->EndRecPtr; - #ifdef WAL_DEBUG if (XLOG_DEBUG || (rmid == RM_XACT_ID && trace_recovery_messages <= DEBUG2) || @@ -1414,8 +1407,8 @@ PerformWalRecovery(void) initStringInfo(&buf); appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ", - LSN_FORMAT_ARGS(ReadRecPtr), - LSN_FORMAT_ARGS(EndRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->EndRecPtr)); xlog_outrec(&buf, xlogreader); appendStringInfoString(&buf, " - "); xlog_outdesc(&buf, xlogreader); @@ -1470,132 +1463,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - /* - * ShmemVariableCache->nextXid must be beyond record's xid. + * Apply the record */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newTLI = ThisTimeLineID; - TimeLineID prevTLI = ThisTimeLineID; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newTLI = checkPoint.ThisTimeLineID; - prevTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newTLI = xlrec.ThisTimeLineID; - prevTLI = xlrec.PrevTimeLineID; - } - - if (newTLI != ThisTimeLineID) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); - - /* Following WAL records should be run with new TLI */ - ThisTimeLineID = newTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->replayEndRecPtr = EndRecPtr; - XLogRecCtl->replayEndTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; - XLogRecCtl->lastReplayedTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = ReadRecPtr; - - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. - */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1654,7 +1525,7 @@ PerformWalRecovery(void) ereport(LOG, (errmsg("redo done at %X/%X system usage: %s", - LSN_FORMAT_ARGS(ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) @@ -1683,6 +1554,148 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record) +{ + XLogRecPtr ReadRecPtr; + XLogRecPtr EndRecPtr; + ErrorContextCallback errcallback; + bool switchedTLI = false; + + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the + * current timeline to change. The record is already considered to + * be part of the new timeline, so we update ThisTimeLineID before + * replaying it. That's important so that replayEndTLI, which is + * recorded as the minimum recovery point's TLI if recovery stops + * after this record, is set correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newTLI = ThisTimeLineID; + TimeLineID prevTLI = ThisTimeLineID; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newTLI = checkPoint.ThisTimeLineID; + prevTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newTLI = xlrec.ThisTimeLineID; + prevTLI = xlrec.PrevTimeLineID; + } + + if (newTLI != ThisTimeLineID) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); + + /* Following WAL records should be run with new TLI */ + ThisTimeLineID = newTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so + * that XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->replayEndRecPtr = EndRecPtr; + XLogRecCtl->replayEndTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we + * see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the + * WAL record are consistent with the existing pages. This check + * is done only if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been + * successfully replayed. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; + XLogRecCtl->lastReplayedTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up + * the receiver so that it notices the updated + * lastReplayedEndRecPtr and sends a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any + * (possibly bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); + + /* + * Wake up any walsenders to notice that we are on a new + * timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------915C630FC23A8D0E9CA95E65-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v6 3/3] Move code to apply one WAL record to a subroutine. @ 2021-09-16 08:07 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-09-16 08:07 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 283 +++++++++++----------- 1 file changed, 148 insertions(+), 135 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index e46215c2586..65edc7e1316 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -363,6 +363,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1361,11 +1362,8 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; - XLogRecPtr ReadRecPtr; - XLogRecPtr EndRecPtr; pg_rusage_init(&ru0); @@ -1387,11 +1385,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - - ReadRecPtr = xlogreader->ReadRecPtr; - EndRecPtr = xlogreader->EndRecPtr; - #ifdef WAL_DEBUG if (XLOG_DEBUG || (rmid == RM_XACT_ID && trace_recovery_messages <= DEBUG2) || @@ -1401,8 +1394,8 @@ PerformWalRecovery(void) initStringInfo(&buf); appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ", - LSN_FORMAT_ARGS(ReadRecPtr), - LSN_FORMAT_ARGS(EndRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->EndRecPtr)); xlog_outrec(&buf, xlogreader); appendStringInfoString(&buf, " - "); xlog_outdesc(&buf, xlogreader); @@ -1457,132 +1450,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - /* - * ShmemVariableCache->nextXid must be beyond record's xid. - */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newTLI = ThisTimeLineID; - TimeLineID prevTLI = ThisTimeLineID; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newTLI = checkPoint.ThisTimeLineID; - prevTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newTLI = xlrec.ThisTimeLineID; - prevTLI = xlrec.PrevTimeLineID; - } - - if (newTLI != ThisTimeLineID) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); - - /* Following WAL records should be run with new TLI */ - ThisTimeLineID = newTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->replayEndRecPtr = EndRecPtr; - XLogRecCtl->replayEndTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecCtl->info_lck); - XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; - XLogRecCtl->lastReplayedTLI = ThisTimeLineID; - SpinLockRelease(&XLogRecCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = ReadRecPtr; - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. + * Apply the record */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1641,7 +1512,7 @@ PerformWalRecovery(void) ereport(LOG, (errmsg("redo done at %X/%X system usage: %s", - LSN_FORMAT_ARGS(ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) @@ -1670,6 +1541,148 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record) +{ + XLogRecPtr ReadRecPtr; + XLogRecPtr EndRecPtr; + ErrorContextCallback errcallback; + bool switchedTLI = false; + + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the + * current timeline to change. The record is already considered to + * be part of the new timeline, so we update ThisTimeLineID before + * replaying it. That's important so that replayEndTLI, which is + * recorded as the minimum recovery point's TLI if recovery stops + * after this record, is set correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newTLI = ThisTimeLineID; + TimeLineID prevTLI = ThisTimeLineID; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newTLI = checkPoint.ThisTimeLineID; + prevTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newTLI = xlrec.ThisTimeLineID; + prevTLI = xlrec.PrevTimeLineID; + } + + if (newTLI != ThisTimeLineID) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(EndRecPtr, newTLI, prevTLI); + + /* Following WAL records should be run with new TLI */ + ThisTimeLineID = newTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so + * that XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->replayEndRecPtr = EndRecPtr; + XLogRecCtl->replayEndTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we + * see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the + * WAL record are consistent with the existing pages. This check + * is done only if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been + * successfully replayed. + */ + SpinLockAcquire(&XLogRecCtl->info_lck); + XLogRecCtl->lastReplayedEndRecPtr = EndRecPtr; + XLogRecCtl->lastReplayedTLI = ThisTimeLineID; + SpinLockRelease(&XLogRecCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up + * the receiver so that it notices the updated + * lastReplayedEndRecPtr and sends a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any + * (possibly bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(EndRecPtr, ThisTimeLineID); + + /* + * Wake up any walsenders to notice that we are on a new + * timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------7E727ABC31D380A53E56FBBC-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v8 4/4] Move code to apply one WAL record to a subroutine. @ 2021-09-16 08:07 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-09-16 08:07 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 267 +++++++++++----------- 1 file changed, 139 insertions(+), 128 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index c21436190fd..72dc611567e 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -367,6 +367,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1398,7 +1399,6 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; @@ -1426,8 +1426,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - if (!StandbyMode) ereport_startup_progress("redo in progress, elapsed time: %ld.%02d s, current LSN: %X/%X", LSN_FORMAT_ARGS(xlogreader->ReadRecPtr)); @@ -1497,133 +1495,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - /* - * ShmemVariableCache->nextXid must be beyond record's xid. - */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newReplayTLI = replayTLI; - TimeLineID prevReplayTLI = replayTLI; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newReplayTLI = checkPoint.ThisTimeLineID; - prevReplayTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newReplayTLI = xlrec.ThisTimeLineID; - prevReplayTLI = xlrec.PrevTimeLineID; - } - - if (newReplayTLI != replayTLI) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(xlogreader->EndRecPtr, newReplayTLI, - prevReplayTLI, replayTLI); - - /* Following WAL records should be run with new TLI */ - replayTLI = newReplayTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecoveryCtl->info_lck); - XLogRecoveryCtl->replayEndRecPtr = xlogreader->EndRecPtr; - XLogRecoveryCtl->replayEndTLI = replayTLI; - SpinLockRelease(&XLogRecoveryCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecoveryCtl->info_lck); - XLogRecoveryCtl->lastReplayedEndRecPtr = xlogreader->EndRecPtr; - XLogRecoveryCtl->lastReplayedTLI = replayTLI; - SpinLockRelease(&XLogRecoveryCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = xlogreader->ReadRecPtr; - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. + * Apply the record */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(xlogreader->EndRecPtr, replayTLI); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record, &replayTLI); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1711,6 +1586,142 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI) +{ + ErrorContextCallback errcallback; + bool switchedTLI = false; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the current + * timeline to change. The record is already considered to be part of the + * new timeline, so we update replayTLI before replaying it. That's + * important so that replayEndTLI, which is recorded as the minimum + * recovery point's TLI if recovery stops after this record, is set + * correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newReplayTLI = *replayTLI; + TimeLineID prevReplayTLI = *replayTLI; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newReplayTLI = checkPoint.ThisTimeLineID; + prevReplayTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newReplayTLI = xlrec.ThisTimeLineID; + prevReplayTLI = xlrec.PrevTimeLineID; + } + + if (newReplayTLI != *replayTLI) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(xlogreader->EndRecPtr, + newReplayTLI, prevReplayTLI, *replayTLI); + + /* Following WAL records should be run with new TLI */ + *replayTLI = newReplayTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so that + * XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecoveryCtl->info_lck); + XLogRecoveryCtl->replayEndRecPtr = xlogreader->EndRecPtr; + XLogRecoveryCtl->replayEndTLI = *replayTLI; + SpinLockRelease(&XLogRecoveryCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the WAL + * record are consistent with the existing pages. This check is done only + * if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been successfully + * replayed. + */ + SpinLockAcquire(&XLogRecoveryCtl->info_lck); + XLogRecoveryCtl->lastReplayedEndRecPtr = xlogreader->EndRecPtr; + XLogRecoveryCtl->lastReplayedTLI = *replayTLI; + SpinLockRelease(&XLogRecoveryCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = xlogreader->ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up the + * receiver so that it notices the updated lastReplayedEndRecPtr and sends + * a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any (possibly + * bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(xlogreader->EndRecPtr, *replayTLI); + + /* + * Wake up any walsenders to notice that we are on a new timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------2135AFA2DD79B96A7476E2B5-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v7 5/5] Move code to apply one WAL record to a subroutine. @ 2021-09-16 08:07 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-09-16 08:07 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 284 +++++++++++----------- 1 file changed, 147 insertions(+), 137 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 5b9d928a8ab..fe6b215b9c5 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -367,6 +367,7 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI); static void xlog_block_info(StringInfo buf, XLogReaderState *record); static void readRecoverySignalFile(void); @@ -1396,11 +1397,8 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; - XLogRecPtr ReadRecPtr; - XLogRecPtr EndRecPtr; pg_rusage_init(&ru0); @@ -1426,14 +1424,9 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - - ReadRecPtr = xlogreader->ReadRecPtr; - EndRecPtr = xlogreader->EndRecPtr; - if (!StandbyMode) ereport_startup_progress("redo in progress, elapsed time: %ld.%02d s, current LSN: %X/%X", - LSN_FORMAT_ARGS(ReadRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr)); #ifdef WAL_DEBUG if (XLOG_DEBUG || @@ -1444,8 +1437,8 @@ PerformWalRecovery(void) initStringInfo(&buf); appendStringInfo(&buf, "REDO @ %X/%X; LSN %X/%X: ", - LSN_FORMAT_ARGS(ReadRecPtr), - LSN_FORMAT_ARGS(EndRecPtr)); + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->EndRecPtr)); xlog_outrec(&buf, xlogreader); appendStringInfoString(&buf, " - "); xlog_outdesc(&buf, xlogreader); @@ -1500,133 +1493,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - /* - * ShmemVariableCache->nextXid must be beyond record's xid. - */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newReplayTLI = replayTLI; - TimeLineID prevReplayTLI = replayTLI; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newReplayTLI = checkPoint.ThisTimeLineID; - prevReplayTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newReplayTLI = xlrec.ThisTimeLineID; - prevReplayTLI = xlrec.PrevTimeLineID; - } - - if (newReplayTLI != replayTLI) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(EndRecPtr, newReplayTLI, - prevReplayTLI, replayTLI); - - /* Following WAL records should be run with new TLI */ - replayTLI = newReplayTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecoveryCtl->info_lck); - XLogRecoveryCtl->replayEndRecPtr = EndRecPtr; - XLogRecoveryCtl->replayEndTLI = replayTLI; - SpinLockRelease(&XLogRecoveryCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - checkXLogConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. - */ - SpinLockAcquire(&XLogRecoveryCtl->info_lck); - XLogRecoveryCtl->lastReplayedEndRecPtr = EndRecPtr; - XLogRecoveryCtl->lastReplayedTLI = replayTLI; - SpinLockRelease(&XLogRecoveryCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = ReadRecPtr; - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. + * Apply the record */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(EndRecPtr, replayTLI); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record, &replayTLI); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1685,7 +1555,7 @@ PerformWalRecovery(void) ereport(LOG, (errmsg("redo done at %X/%X system usage: %s", - LSN_FORMAT_ARGS(ReadRecPtr), + LSN_FORMAT_ARGS(xlogreader->ReadRecPtr), pg_rusage_show(&ru0)))); xtime = GetLatestXTime(); if (xtime) @@ -1714,6 +1584,146 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI) +{ + XLogRecPtr ReadRecPtr; + XLogRecPtr EndRecPtr; + ErrorContextCallback errcallback; + bool switchedTLI = false; + + ReadRecPtr = xlogreader->ReadRecPtr; + EndRecPtr = xlogreader->EndRecPtr; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the current + * timeline to change. The record is already considered to be part of the + * new timeline, so we update replayTLI before replaying it. That's + * important so that replayEndTLI, which is recorded as the minimum + * recovery point's TLI if recovery stops after this record, is set + * correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newReplayTLI = *replayTLI; + TimeLineID prevReplayTLI = *replayTLI; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newReplayTLI = checkPoint.ThisTimeLineID; + prevReplayTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newReplayTLI = xlrec.ThisTimeLineID; + prevReplayTLI = xlrec.PrevTimeLineID; + } + + if (newReplayTLI != *replayTLI) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(EndRecPtr, newReplayTLI, prevReplayTLI, *replayTLI); + + /* Following WAL records should be run with new TLI */ + *replayTLI = newReplayTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so that + * XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecoveryCtl->info_lck); + XLogRecoveryCtl->replayEndRecPtr = EndRecPtr; + XLogRecoveryCtl->replayEndTLI = *replayTLI; + SpinLockRelease(&XLogRecoveryCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the WAL + * record are consistent with the existing pages. This check is done only + * if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + checkXLogConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been successfully + * replayed. + */ + SpinLockAcquire(&XLogRecoveryCtl->info_lck); + XLogRecoveryCtl->lastReplayedEndRecPtr = EndRecPtr; + XLogRecoveryCtl->lastReplayedTLI = *replayTLI; + SpinLockRelease(&XLogRecoveryCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up the + * receiver so that it notices the updated lastReplayedEndRecPtr and sends + * a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any (possibly + * bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(EndRecPtr, *replayTLI); + + /* + * Wake up any walsenders to notice that we are on a new timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Error context callback for errors occurring during rm_redo(). */ -- 2.30.2 --------------3904592D294EBE853403D5C1-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2021-10-28 01:50 Michael Paquier <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Michael Paquier @ 2021-10-28 01:50 UTC (permalink / raw) To: [email protected] Add TAP test for archive_cleanup_command and recovery_end_command This adds tests checking for the execution of both commands. The recovery test 002_archiving.pl is nicely adapted to that, as promotion is triggered already twice there, and even if any of those commands fail they don't affect recovery or promotion. A command success is checked using a file generated by an "echo" command, that should be able to work in all the buildfarm environments, even Msys (but we'll know soon about that). Command failure is tested with an "echo" command that points to a path that does not exist, scanning the backend logs to make sure that the failure happens. Both rely on the backend triggering the commands from the root of the data folder, making its logic more robust. Thanks to Neha Sharma for the extra tests on Windows. Author: Amul Sul, Michael Paquier Reviewed-by: Andres Freund, Euler Taveira Discussion: https://postgr.es/m/CAAJ_b95R_c4T5moq30qsybSU=eDzDHm=4SPiAWaiMWc2OW7=1Q@mail.gmail.com Branch ------ master Details ------- https://git.postgresql.org/pg/commitdiff/46dea2419ee7895a4eb3d048317682e6f18a17e1 Modified Files -------------- src/test/recovery/t/002_archiving.pl | 47 +++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v9 5/5] Move code to apply one WAL record to a subroutine. @ 2021-12-17 07:00 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2021-12-17 07:00 UTC (permalink / raw) --- src/backend/access/transam/xlogrecovery.c | 282 +++++++++++----------- 1 file changed, 147 insertions(+), 135 deletions(-) diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index b9fb61d1dbb..875188cc7a8 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -374,6 +374,8 @@ static char recoveryStopName[MAXFNAMELEN]; static bool recoveryStopAfter; /* prototypes for local functions */ +static void ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI); + static void readRecoverySignalFile(void); static void validateRecoveryParameters(void); static bool read_backup_label(XLogRecPtr *checkPointLoc, @@ -1569,7 +1571,6 @@ PerformWalRecovery(void) if (record != NULL) { - ErrorContextCallback errcallback; TimestampTz xtime; PGRUsage ru0; @@ -1597,8 +1598,6 @@ PerformWalRecovery(void) */ do { - bool switchedTLI = false; - if (!StandbyMode) ereport_startup_progress("redo in progress, elapsed time: %ld.%02d s, current LSN: %X/%X", LSN_FORMAT_ARGS(xlogreader->ReadRecPtr)); @@ -1668,140 +1667,10 @@ PerformWalRecovery(void) recoveryPausesHere(false); } - /* Setup error traceback support for ereport() */ - errcallback.callback = rm_redo_error_callback; - errcallback.arg = (void *) xlogreader; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - /* - * ShmemVariableCache->nextXid must be beyond record's xid. - */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); - - /* - * Before replaying this record, check if this record causes the - * current timeline to change. The record is already considered to - * be part of the new timeline, so we update ThisTimeLineID before - * replaying it. That's important so that replayEndTLI, which is - * recorded as the minimum recovery point's TLI if recovery stops - * after this record, is set correctly. - */ - if (record->xl_rmid == RM_XLOG_ID) - { - TimeLineID newReplayTLI = replayTLI; - TimeLineID prevReplayTLI = replayTLI; - uint8 info = record->xl_info & ~XLR_INFO_MASK; - - if (info == XLOG_CHECKPOINT_SHUTDOWN) - { - CheckPoint checkPoint; - - memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); - newReplayTLI = checkPoint.ThisTimeLineID; - prevReplayTLI = checkPoint.PrevTimeLineID; - } - else if (info == XLOG_END_OF_RECOVERY) - { - xl_end_of_recovery xlrec; - - memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); - newReplayTLI = xlrec.ThisTimeLineID; - prevReplayTLI = xlrec.PrevTimeLineID; - } - - if (newReplayTLI != replayTLI) - { - /* Check that it's OK to switch to this TLI */ - checkTimeLineSwitch(xlogreader->EndRecPtr, newReplayTLI, - prevReplayTLI, replayTLI); - - /* Following WAL records should be run with new TLI */ - replayTLI = newReplayTLI; - switchedTLI = true; - } - } - - /* - * Update shared replayEndRecPtr before replaying this record, so - * that XLogFlush will update minRecoveryPoint correctly. - */ - SpinLockAcquire(&XLogRecoveryCtl->info_lck); - XLogRecoveryCtl->replayEndRecPtr = xlogreader->EndRecPtr; - XLogRecoveryCtl->replayEndTLI = replayTLI; - SpinLockRelease(&XLogRecoveryCtl->info_lck); - - /* - * If we are attempting to enter Hot Standby mode, process XIDs we - * see - */ - if (standbyState >= STANDBY_INITIALIZED && - TransactionIdIsValid(record->xl_xid)) - RecordKnownAssignedTransactionIds(record->xl_xid); - - /* - * Some XLOG record types that are related to recovery are - * processed directly here, rather than in xlog_redo() - */ - if (record->xl_rmid == RM_XLOG_ID) - xlogrecovery_redo(xlogreader, replayTLI); - - /* Now apply the WAL record itself */ - RmgrTable[record->xl_rmid].rm_redo(xlogreader); - - /* - * After redo, check whether the backup pages associated with the - * WAL record are consistent with the existing pages. This check - * is done only if consistency check is enabled for this record. - */ - if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) - verifyBackupPageConsistency(xlogreader); - - /* Pop the error context stack */ - error_context_stack = errcallback.previous; - /* - * Update lastReplayedEndRecPtr after this record has been - * successfully replayed. + * Apply the record */ - SpinLockAcquire(&XLogRecoveryCtl->info_lck); - XLogRecoveryCtl->lastReplayedEndRecPtr = xlogreader->EndRecPtr; - XLogRecoveryCtl->lastReplayedTLI = replayTLI; - SpinLockRelease(&XLogRecoveryCtl->info_lck); - - /* Also remember its starting position. */ - LastReplayedReadRecPtr = xlogreader->ReadRecPtr; - - /* - * If rm_redo called XLogRequestWalReceiverReply, then we wake up - * the receiver so that it notices the updated - * lastReplayedEndRecPtr and sends a reply to the primary. - */ - if (doRequestWalReceiverReply) - { - doRequestWalReceiverReply = false; - WalRcvForceReply(); - } - - /* Allow read-only connections if we're consistent now */ - CheckRecoveryConsistency(); - - /* Is this a timeline switch? */ - if (switchedTLI) - { - /* - * Before we continue on the new timeline, clean up any - * (possibly bogus) future WAL segments on the old timeline. - */ - RemoveNonParentXlogFiles(xlogreader->EndRecPtr, replayTLI); - - /* - * Wake up any walsenders to notice that we are on a new - * timeline. - */ - if (AllowCascadeReplication()) - WalSndWakeup(); - } + ApplyWalRecord(xlogreader, record, &replayTLI); /* Exit loop if we reached inclusive recovery target */ if (recoveryStopsAfter(xlogreader)) @@ -1889,6 +1758,149 @@ PerformWalRecovery(void) (errmsg("recovery ended before configured recovery target was reached"))); } +/* + * Subroutine of PerformWalRecovery, to apply one WAL record. + */ +static void +ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *replayTLI) +{ + ErrorContextCallback errcallback; + bool switchedTLI = false; + + /* Setup error traceback support for ereport() */ + errcallback.callback = rm_redo_error_callback; + errcallback.arg = (void *) xlogreader; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* + * ShmemVariableCache->nextXid must be beyond record's xid. + */ + AdvanceNextFullTransactionIdPastXid(record->xl_xid); + + /* + * Before replaying this record, check if this record causes the current + * timeline to change. The record is already considered to be part of the + * new timeline, so we update replayTLI before replaying it. That's + * important so that replayEndTLI, which is recorded as the minimum + * recovery point's TLI if recovery stops after this record, is set + * correctly. + */ + if (record->xl_rmid == RM_XLOG_ID) + { + TimeLineID newReplayTLI = *replayTLI; + TimeLineID prevReplayTLI = *replayTLI; + uint8 info = record->xl_info & ~XLR_INFO_MASK; + + if (info == XLOG_CHECKPOINT_SHUTDOWN) + { + CheckPoint checkPoint; + + memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint)); + newReplayTLI = checkPoint.ThisTimeLineID; + prevReplayTLI = checkPoint.PrevTimeLineID; + } + else if (info == XLOG_END_OF_RECOVERY) + { + xl_end_of_recovery xlrec; + + memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery)); + newReplayTLI = xlrec.ThisTimeLineID; + prevReplayTLI = xlrec.PrevTimeLineID; + } + + if (newReplayTLI != *replayTLI) + { + /* Check that it's OK to switch to this TLI */ + checkTimeLineSwitch(xlogreader->EndRecPtr, + newReplayTLI, prevReplayTLI, *replayTLI); + + /* Following WAL records should be run with new TLI */ + *replayTLI = newReplayTLI; + switchedTLI = true; + } + } + + /* + * Update shared replayEndRecPtr before replaying this record, so that + * XLogFlush will update minRecoveryPoint correctly. + */ + SpinLockAcquire(&XLogRecoveryCtl->info_lck); + XLogRecoveryCtl->replayEndRecPtr = xlogreader->EndRecPtr; + XLogRecoveryCtl->replayEndTLI = *replayTLI; + SpinLockRelease(&XLogRecoveryCtl->info_lck); + + /* + * If we are attempting to enter Hot Standby mode, process XIDs we see + */ + if (standbyState >= STANDBY_INITIALIZED && + TransactionIdIsValid(record->xl_xid)) + RecordKnownAssignedTransactionIds(record->xl_xid); + + /* + * Some XLOG record types that are related to recovery are processed + * directly here, rather than in xlog_redo() + */ + if (record->xl_rmid == RM_XLOG_ID) + xlogrecovery_redo(xlogreader, *replayTLI); + + /* Now apply the WAL record itself */ + RmgrTable[record->xl_rmid].rm_redo(xlogreader); + + /* + * After redo, check whether the backup pages associated with the WAL + * record are consistent with the existing pages. This check is done only + * if consistency check is enabled for this record. + */ + if ((record->xl_info & XLR_CHECK_CONSISTENCY) != 0) + verifyBackupPageConsistency(xlogreader); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + + /* + * Update lastReplayedEndRecPtr after this record has been successfully + * replayed. + */ + SpinLockAcquire(&XLogRecoveryCtl->info_lck); + XLogRecoveryCtl->lastReplayedEndRecPtr = xlogreader->EndRecPtr; + XLogRecoveryCtl->lastReplayedTLI = *replayTLI; + SpinLockRelease(&XLogRecoveryCtl->info_lck); + + /* Also remember its starting position. */ + LastReplayedReadRecPtr = xlogreader->ReadRecPtr; + + /* + * If rm_redo called XLogRequestWalReceiverReply, then we wake up the + * receiver so that it notices the updated lastReplayedEndRecPtr and sends + * a reply to the primary. + */ + if (doRequestWalReceiverReply) + { + doRequestWalReceiverReply = false; + WalRcvForceReply(); + } + + /* Allow read-only connections if we're consistent now */ + CheckRecoveryConsistency(); + + /* Is this a timeline switch? */ + if (switchedTLI) + { + /* + * Before we continue on the new timeline, clean up any (possibly + * bogus) future WAL segments on the old timeline. + */ + RemoveNonParentXlogFiles(xlogreader->EndRecPtr, *replayTLI); + + /* + * Wake up any walsenders to notice that we are on a new timeline. + */ + if (AllowCascadeReplication()) + WalSndWakeup(); + } +} + /* * Some XLOG RM record types that are directly related to WAL recovery are * handled here rather than in the xlog_redo() -- 2.30.2 --------------24AFD7B565D1554637A9E916-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-11 06:48 Thomas Munro <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 2 replies; 21+ messages in thread From: Thomas Munro @ 2022-04-11 06:48 UTC (permalink / raw) To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Michael Paquier <[email protected]>; PostgreSQL Hackers <[email protected]> On Sat, Apr 9, 2022 at 12:59 PM Andres Freund <[email protected]> wrote: > On 2022-04-08 17:55:51 -0400, Tom Lane wrote: > > I tried adjusting the patch so it does guarantee that (as attached), > > and in two out of two tries it got past the archive_cleanup_command > > failure but then hung up waiting for standby2 to promote. > > Adding > > $node_standby->safe_psql('postgres', "SELECT pg_switch_wal()"); > just after > $node_standby2->start; > > makes the tests pass here. Sorry for the delay... I got a bit confused about the different things going on in this thread but I hope I've got it now: 1. This test had some pre-existing bugs/races, which hadn't failed before due to scheduling, even under Valgrind. The above changes appear to fix those problems. To Michael for comment. > What is that second test really testing? > > # Check the presence of temporary files specifically generated during > # archive recovery. To ensure the presence of the temporary history > # file, switch to a timeline large enough to allow a standby to recover > # a history file from an archive. As this requires at least two timeline > # switches, promote the existing standby first. Then create a second > # standby based on the promoted one. Finally, the second standby is > # promoted. > > Note "Then create a second standby based on the promoted one." - but that's > not actually what's happening: 2. There may also be other problems with the test but those aren't relevant to skink's failure, which starts on the 5th test. To Michael for comment. > But I think there's also something funky in the prefetching logic. I think it > may attempt restoring during prefetching somehow, even though there's code > that appears to try to prevent that? 3. Urghl. Yeah. There is indeed code to report XLREAD_WOULDBLOCK for the lastSourceFailed case, but we don't really want to do that more often than every 5 seconds. So I think I need to make that "sticky", so that we don't attempt to prefetch again (ie to read from the next segment, which invokes restore_command) until we've replayed all the records we already have, then hit the end and go to sleep. The attached patch does that, and makes the offending test pass under Valgrind for me, even without the other changes already mentioned. If I understand correctly, this is due to a timing race in the tests (though I didn't check where exactly), because all those extra fork/exec calls are extremely slow under Valgrind. > And interestingly I'm not seeing the > "switched WAL source from stream to archive after failure" > lines I'd expect. I see them now. It's because it gives up when it's reading ahead (nonblocking), which may not be strictly necessary but I found it simpler to think about. Then when it tries again in 5 seconds it's in blocking mode so it doesn't give up so easily. 2022-04-11 18:15:08.220 NZST [524796] DEBUG: switched WAL source from stream to archive after failure cp: cannot stat '/tmp/archive/000000010000000000000017': No such file or directory 2022-04-11 18:15:08.226 NZST [524796] DEBUG: could not restore file "000000010000000000000017" from archive: child process exited with exit code 1 2022-04-11 18:15:08.226 NZST [524796] DEBUG: could not open file "pg_wal/000000010000000000000017": No such file or directory 2022-04-11 18:15:08.226 NZST [524796] DEBUG: switched WAL source from archive to stream after failure Attachments: [text/x-patch] 0001-Don-t-retry-restore_command-while-reading-ahead.patch (2.5K, ../../CA+hUKGJ-gvLg9vJgMvu62XW-Nsb2Hf40DvqmKdusVeJ7KV-crA@mail.gmail.com/2-0001-Don-t-retry-restore_command-while-reading-ahead.patch) download | inline diff: From e488e20f7c364ba1adaae8d1f6ea92a7f5d0bda6 Mon Sep 17 00:00:00 2001 From: Thomas Munro <[email protected]> Date: Mon, 11 Apr 2022 17:16:24 +1200 Subject: [PATCH] Don't retry restore_command while reading ahead. Suppress further attempts to read ahead in the WAL if we run out of data, until records already decoded have been replayed. When replaying from archives, this avoids repeatedly retrying the restore_command until after we've slept for 5 seconds. When replaying from a network stream, this makes us less aggressive in theory, but we probably can't benefit from being more aggressive yet anyway as the flushedUpto variable doesn't advance until we run out of data (though we could improve that in future). While here, fix a potential off-by one confusion with the no_readahead_until mechanism: we suppress readahead until after the record that begins at that LSN has been replayed (ie we are replaying a higher LSN), so change < to <=. Defect in commit 5dc0418f. Reported-by: Andres Freund <[email protected]> Discussion: https://postgr.es/m/20220409005910.alw46xqmmgny2sgr%40alap3.anarazel.de --- src/backend/access/transam/xlogprefetcher.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/access/transam/xlogprefetcher.c b/src/backend/access/transam/xlogprefetcher.c index f3428888d2..43e82b65c2 100644 --- a/src/backend/access/transam/xlogprefetcher.c +++ b/src/backend/access/transam/xlogprefetcher.c @@ -487,8 +487,8 @@ XLogPrefetcherNextBlock(uintptr_t pgsr_private, XLogRecPtr *lsn) */ nonblocking = XLogReaderHasQueuedRecordOrError(reader); - /* Certain records act as barriers for all readahead. */ - if (nonblocking && replaying_lsn < prefetcher->no_readahead_until) + /* Readahead is disabled until we replay past a certain point. */ + if (nonblocking && replaying_lsn <= prefetcher->no_readahead_until) return LRQ_NEXT_AGAIN; record = XLogReadAhead(prefetcher->reader, nonblocking); @@ -496,8 +496,13 @@ XLogPrefetcherNextBlock(uintptr_t pgsr_private, XLogRecPtr *lsn) { /* * We can't read any more, due to an error or lack of data in - * nonblocking mode. + * nonblocking mode. Don't try to read ahead again until we've + * replayed everything already decoded. */ + if (nonblocking && prefetcher->reader->decode_queue_tail) + prefetcher->no_readahead_until = + prefetcher->reader->decode_queue_tail->lsn; + return LRQ_NEXT_AGAIN; } -- 2.30.2 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-11 07:43 Michael Paquier <[email protected]> parent: Thomas Munro <[email protected]> 1 sibling, 0 replies; 21+ messages in thread From: Michael Paquier @ 2022-04-11 07:43 UTC (permalink / raw) To: Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, Apr 11, 2022 at 06:48:58PM +1200, Thomas Munro wrote: > Sorry for the delay... I got a bit confused about the different things > going on in this thread but I hope I've got it now: > > 1. This test had some pre-existing bugs/races, which hadn't failed > before due to scheduling, even under Valgrind. The above changes > appear to fix those problems. To Michael for comment. I have seen the thread, and there is a lot in it. I will try to look tomorrow at the parts I got involved in. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-12 03:49 Michael Paquier <[email protected]> parent: Thomas Munro <[email protected]> 1 sibling, 2 replies; 21+ messages in thread From: Michael Paquier @ 2022-04-12 03:49 UTC (permalink / raw) To: Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, Apr 11, 2022 at 06:48:58PM +1200, Thomas Munro wrote: > 1. This test had some pre-existing bugs/races, which hadn't failed > before due to scheduling, even under Valgrind. The above changes > appear to fix those problems. To Michael for comment. Yeah, there are two problems here. From what I can see, ensuring the execution of archive_cleanup_command on the standby needs the checkpoint on the primary and the restart point on the standby. So pg_current_wal_lsn() should be located after the primary's checkpoint and not before it so as we are sure that the checkpoint records finds its way to the standby. That's what Tom mentioned upthread. The second problem is to make sure that $standby2 sees the promotion of $standby and its history file, but we also want to recover 00000002.history from some archives to create a RECOVERYHISTORY at recovery for the purpose of the test. Switching to a new segment as proposed by Andres does not seem completely right to me because we are not 100% sure of the ordering an archive is going to happen, no? I think that the logic to create $standby2 from the initial backup of the primary is right, because there is no 00000002.history in it, but we also need to be sure that 00000002.history has been archived once the promotion of $standby is done. This can be validated thanks to the logs, actually. >> What is that second test really testing? >> >> # Check the presence of temporary files specifically generated during >> # archive recovery. To ensure the presence of the temporary history >> # file, switch to a timeline large enough to allow a standby to recover >> # a history file from an archive. As this requires at least two timeline >> # switches, promote the existing standby first. Then create a second >> # standby based on the promoted one. Finally, the second standby is >> # promoted. >> >> Note "Then create a second standby based on the promoted one." - but that's >> not actually what's happening: > > 2. There may also be other problems with the test but those aren't > relevant to skink's failure, which starts on the 5th test. To Michael > for comment. This comes from df86e52, where we want to recovery a history file that would be created as RECOVERYHISTORY and make sure that the file gets removed at the end of recovery. So $standby2 should choose a new timeline different from the one of chosen by $standby. Looking back at what has been done, it seems to me that the comment is the incorrect part: https://www.postgresql.org/message-id/[email protected] All that stuff leads me to the attached. Thoughts? -- Michael Attachments: [text/x-diff] tap-archiving-michael.patch (3.5K, ../../[email protected]/2-tap-archiving-michael.patch) download | inline diff: diff --git a/src/test/recovery/t/002_archiving.pl b/src/test/recovery/t/002_archiving.pl index c8f5ffbaf0..45aafcb35c 100644 --- a/src/test/recovery/t/002_archiving.pl +++ b/src/test/recovery/t/002_archiving.pl @@ -24,6 +24,8 @@ $node_primary->backup($backup_name); # Initialize standby node from backup, fetching WAL from archives my $node_standby = PostgreSQL::Test::Cluster->new('standby'); +# Note that this makes the standby archive its contents on the archives +# of the primary. $node_standby->init_from_backup($node_primary, $backup_name, has_restoring => 1); $node_standby->append_conf('postgresql.conf', @@ -44,13 +46,16 @@ $node_standby->start; # Create some content on primary $node_primary->safe_psql('postgres', "CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a"); -my $current_lsn = - $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); # Note the presence of this checkpoint for the archive_cleanup_command # check done below, before switching to a new segment. $node_primary->safe_psql('postgres', "CHECKPOINT"); +# Done after the checkpoint to ensure that the checkpoint gets replayed +# on the standby. +my $current_lsn = + $node_primary->safe_psql('postgres', "SELECT pg_current_wal_lsn();"); + # Force archiving of WAL file to make it present on primary $node_primary->safe_psql('postgres', "SELECT pg_switch_wal()"); @@ -81,10 +86,34 @@ ok( !-f "$data_dir/$recovery_end_command_file", # file, switch to a timeline large enough to allow a standby to recover # a history file from an archive. As this requires at least two timeline # switches, promote the existing standby first. Then create a second -# standby based on the promoted one. Finally, the second standby is -# promoted. +# standby based on the primary, using its archives. Finally, the second +# standby is promoted. $node_standby->promote; +# Wait until the history file has been archived on the archives of the +# primary once the promotion of the standby completes. +my $primary_archive = $node_primary->archive_dir; +my $max_attempts = 10 * $PostgreSQL::Test::Utils::timeout_default; +my $attempts = 0; +while ($attempts < $max_attempts) +{ + last if (-e "$primary_archive/00000002.history"); + + # Wait 0.1 second before retrying. + usleep(100_000); + + $attempts++; +} + +if ($attempts >= $max_attempts) +{ + die "timed out waiting for 00000002.history\n"; +} +else +{ + note "found 00000002.history after $attempts attempts\n" +} + # recovery_end_command should have been triggered on promotion. ok( -f "$data_dir/$recovery_end_command_file", 'recovery_end_command executed after promotion'); @@ -108,14 +137,19 @@ my $log_location = -s $node_standby2->logfile; # Now promote standby2, and check that temporary files specifically # generated during archive recovery are removed by the end of recovery. $node_standby2->promote; + +# Check the logs of the standby to see that the commands have failed. +my $log_contents = slurp_file($node_standby2->logfile, $log_location); my $node_standby2_data = $node_standby2->data_dir; + +like( + $log_contents, + qr/restored log file "00000002.history" from archive/s, + "00000002.history retrieved from the archives"); ok( !-f "$node_standby2_data/pg_wal/RECOVERYHISTORY", "RECOVERYHISTORY removed after promotion"); ok( !-f "$node_standby2_data/pg_wal/RECOVERYXLOG", "RECOVERYXLOG removed after promotion"); - -# Check the logs of the standby to see that the commands have failed. -my $log_contents = slurp_file($node_standby2->logfile, $log_location); like( $log_contents, qr/WARNING:.*recovery_end_command/s, [application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc) download ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-16 20:56 Thomas Munro <[email protected]> parent: Michael Paquier <[email protected]> 1 sibling, 1 reply; 21+ messages in thread From: Thomas Munro @ 2022-04-16 20:56 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Tue, Apr 12, 2022 at 3:49 PM Michael Paquier <[email protected]> wrote: > All that stuff leads me to the attached. Thoughts? Under valgrind I got "Undefined subroutine &main::usleep called at t/002_archiving.pl line 103" so I added "use Time::HiRes qw(usleep);", and now I get past the first 4 tests with your patch, but then promotion times out, not sure why: +++ tap check in src/test/recovery +++ t/002_archiving.pl .. ok 1 - check content from archives ok 2 - archive_cleanup_command executed on checkpoint ok 3 - recovery_end_command not executed yet # found 00000002.history after 14 attempts ok 4 - recovery_end_command executed after promotion Bailout called. Further testing stopped: command "pg_ctl -D /home/tmunro/projects/postgresql/src/test/recovery/tmp_check/t_002_archiving_standby2_data/pgdata -l /home/tmunro/projects/postgresql/src/test/recovery/tmp_check/log/002_archiving_standby2.log promote" exited with value 1 Since it's quite painful to run TAP tests under valgrind, I found a place to stick a plain old sleep to repro these problems: --- a/src/test/perl/PostgreSQL/Test/Cluster.pm +++ b/src/test/perl/PostgreSQL/Test/Cluster.pm @@ -1035,7 +1035,7 @@ sub enable_restoring my $copy_command = $PostgreSQL::Test::Utils::windows_os ? qq{copy "$path\\\\%f" "%p"} - : qq{cp "$path/%f" "%p"}; + : qq{sleep 1 && cp "$path/%f" "%p"}; Soon I'll push the fix to the slowness that xlogprefetcher.c accidentally introduced to continuous archive recovery, ie the problem of calling a failing restore_command repeatedly as we approach the end of a WAL segment instead of just once every 5 seconds after we run out of data, and after that you'll probably need to revert that fix locally to repro this. ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-17 04:17 Michael Paquier <[email protected]> parent: Thomas Munro <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Michael Paquier @ 2022-04-17 04:17 UTC (permalink / raw) To: Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Sun, Apr 17, 2022 at 08:56:33AM +1200, Thomas Munro wrote: > Under valgrind I got "Undefined subroutine &main::usleep called at > t/002_archiving.pl line 103" so I added "use Time::HiRes qw(usleep);", > and now I get past the first 4 tests with your patch, but then > promotion times out, not sure why: > > +++ tap check in src/test/recovery +++ > t/002_archiving.pl .. > ok 1 - check content from archives > ok 2 - archive_cleanup_command executed on checkpoint > ok 3 - recovery_end_command not executed yet > # found 00000002.history after 14 attempts > ok 4 - recovery_end_command executed after promotion > Bailout called. Further testing stopped: command "pg_ctl -D > /home/tmunro/projects/postgresql/src/test/recovery/tmp_check/t_002_archiving_standby2_data/pgdata > -l /home/tmunro/projects/postgresql/src/test/recovery/tmp_check/log/002_archiving_standby2.log > promote" exited with value 1 Hmm. As far as I can see, aren't you just hitting the 60s timeout of pg_ctl here due to the slowness of valgrind? > Since it's quite painful to run TAP tests under valgrind, I found a > place to stick a plain old sleep to repro these problems: Actually, I am wondering how you are patching Cluster.pm to do that. > Soon I'll push the fix to the slowness that xlogprefetcher.c > accidentally introduced to continuous archive recovery, ie the problem > of calling a failing restore_command repeatedly as we approach the end > of a WAL segment instead of just once every 5 seconds after we run out > of data, and after that you'll probably need to revert that fix > locally to repro this. Okay. Thanks. Anyway, I'll do something about that tomorrow (no room to look at the buildfarm today), and I was thinking about replacing the while loop I had in the last version of the patch with a poll_query_until that does a pg_stat_file() with an absolute path to the history file to avoid the dependency to usleep() in the test, splitting the fix into two commits as there is more than one problem, each applying to different branches. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-17 14:56 Andrew Dunstan <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Andrew Dunstan @ 2022-04-17 14:56 UTC (permalink / raw) To: Michael Paquier <[email protected]>; Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On 2022-04-17 Su 00:17, Michael Paquier wrote: >> Since it's quite painful to run TAP tests under valgrind, I found a >> place to stick a plain old sleep to repro these problems: > Actually, I am wondering how you are patching Cluster.pm to do that. I don't really think it's Cluster.pm's business to deal with that. It takes an install path as given either explicitly or implicitly. It shouldn't be too hard to get Makefile.global to install valgrind wrappers into the tmp_install/bin directory. cheers andrew -- Andrew Dunstan EDB: https://www.enterprisedb.com ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-17 23:49 Michael Paquier <[email protected]> parent: Andrew Dunstan <[email protected]> 0 siblings, 2 replies; 21+ messages in thread From: Michael Paquier @ 2022-04-17 23:49 UTC (permalink / raw) To: Andrew Dunstan <[email protected]>; +Cc: Thomas Munro <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Sun, Apr 17, 2022 at 10:56:08AM -0400, Andrew Dunstan wrote: > I don't really think it's Cluster.pm's business to deal with that. It > takes an install path as given either explicitly or implicitly. > > It shouldn't be too hard to get Makefile.global to install valgrind > wrappers into the tmp_install/bin directory. Or what gets used in just a wrapper of the contents of bin/ that get enforced to be first in PATH? -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../YlynfULh%[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-18 04:55 Michael Paquier <[email protected]> parent: Michael Paquier <[email protected]> 1 sibling, 0 replies; 21+ messages in thread From: Michael Paquier @ 2022-04-18 04:55 UTC (permalink / raw) To: Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Tue, Apr 12, 2022 at 12:49:48PM +0900, Michael Paquier wrote: > This comes from df86e52, where we want to recovery a history file that > would be created as RECOVERYHISTORY and make sure that the file gets > removed at the end of recovery. So $standby2 should choose a new > timeline different from the one of chosen by $standby. Looking back > at what has been done, it seems to me that the comment is the > incorrect part: > https://www.postgresql.org/message-id/[email protected] acf1dd42 has taken care of the failures of this test with skink, and I have just taken care of the two races in the tests with e61efaf and 1a8b110. I have left e61efaf out of REL_10_STABLE as the idea of relying on a poll_query_until() with pg_stat_file() and an absolute path would not work there, and the branch will be EOL'd soon while there were no complains with this test for two years. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../YlzvMeOnn%[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-18 17:32 Andrew Dunstan <[email protected]> parent: Michael Paquier <[email protected]> 1 sibling, 0 replies; 21+ messages in thread From: Andrew Dunstan @ 2022-04-18 17:32 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Thomas Munro <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On 2022-04-17 Su 19:49, Michael Paquier wrote: > On Sun, Apr 17, 2022 at 10:56:08AM -0400, Andrew Dunstan wrote: >> I don't really think it's Cluster.pm's business to deal with that. It >> takes an install path as given either explicitly or implicitly. >> >> It shouldn't be too hard to get Makefile.global to install valgrind >> wrappers into the tmp_install/bin directory. > Or what gets used in just a wrapper of the contents of bin/ that get > enforced to be first in PATH? That seems likely to be difficult. For example pg_ctl might find its colocated postgres rather than the valgrind wrapper. Ditto initdb. cheers andrew -- Andrew Dunstan EDB: https://www.enterprisedb.com ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-18 21:45 Thomas Munro <[email protected]> parent: Michael Paquier <[email protected]> 1 sibling, 1 reply; 21+ messages in thread From: Thomas Munro @ 2022-04-18 21:45 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, Apr 18, 2022 at 11:49 AM Michael Paquier <[email protected]> wrote: > On Sun, Apr 17, 2022 at 10:56:08AM -0400, Andrew Dunstan wrote: > > I don't really think it's Cluster.pm's business to deal with that. It > > takes an install path as given either explicitly or implicitly. > > > > It shouldn't be too hard to get Makefile.global to install valgrind > > wrappers into the tmp_install/bin directory. > > Or what gets used in just a wrapper of the contents of bin/ that get > enforced to be first in PATH? Delayed response to the question on how I did that, because it was a 4 day weekend down here and I got distracted by sunshine... A horrible slow way to do it is to build with -DUSE_VALGRIND and then just run the whole process tree (eg make, perl, psql, ... and all) under valgrind with --trace-children=yes: tmunro@x1:~/projects/postgresql$ valgrind --quiet --suppressions=`pwd`/src/tools/valgrind.supp --trace-children=yes --track-origins=yes --run-libc-freeres=no --vgdb=no --error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END make -C src/test/recovery/ check PROVE_TESTS=t/002_* PROVE_FLAGS=-v I think that sort of thing actually worked when I tried it on a beefier workstation, but it sent my Thinkpad that "only" has a 16GB of RAM into some kind of death spiral. The way I succeeded was indeed using a wrapper script, based on a suggestion from Andres, my kludgy-hardcoded-path-assuming implementation of which looked like: === install-postgres-valgrind, to be run once after building === #!/bin/sh SRC=$HOME/projects/postgresql # move the real binary out of the way mv $SRC/src/backend/postgres $SRC/src/backend/postgres.real # install the wrapper, in the location it'll be copied from for tmp_install cp postgres.valgrind $SRC/src/backend/postgres === === postgres.valgrind wrapper script === #!/bin/sh exec /usr/bin/valgrind \ --quiet \ --error-exitcode=128 \ --suppressions=$HOME/projects/postgresql/src/tools/valgrind.supp \ --trace-children=yes --track-origins=yes --read-var-info=no \ --leak-check=no \ --run-libc-freeres=no \ --vgdb=no \ --error-markers=VALGRINDERROR-BEGIN,VALGRINDERROR-END \ $HOME/projects/postgresql/src/backend/postgres.real \ "$@" === Then just: make -C src/test/recovery/ check PROVE_TESTS=t/002_* PROVE_FLAGS=-v Yeah, it might be quite neat to find a tool-supported way to do that. Tangentially, I'd also like to look into making PostgreSQL-under-Valgrind work on FreeBSD and macOS, which didn't work last time I tried it for reasons that might, I hope, have been fixed on the Valgrind side by now. ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman @ 2022-04-19 02:33 Michael Paquier <[email protected]> parent: Thomas Munro <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Michael Paquier @ 2022-04-19 02:33 UTC (permalink / raw) To: Thomas Munro <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]> On Tue, Apr 19, 2022 at 09:45:11AM +1200, Thomas Munro wrote: > Delayed response to the question on how I did that, because it was a 4 > day weekend down here and I got distracted by sunshine... Happy Easter. > I think that sort of thing actually worked when I tried it on a > beefier workstation, but it sent my Thinkpad that "only" has a 16GB of > RAM into some kind of death spiral. The way I succeeded was indeed > using a wrapper script, based on a suggestion from Andres, my > kludgy-hardcoded-path-assuming implementation of which looked like: > > Yeah, it might be quite neat to find a tool-supported way to do that. Thanks for the details. I feared that it was something like that for the backend. At least that's better than having valgrind spawn all the processes kicked by the make command. :/ > Tangentially, I'd also like to look into making > PostgreSQL-under-Valgrind work on FreeBSD and macOS, which didn't work > last time I tried it for reasons that might, I hope, have been fixed > on the Valgrind side by now. Okay. As a side note, skink has cooled down since acf1dd4, and did not complain either after the additions of e61efaf and 1a8b110. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v7 08/13] Remove table_scan_bitmap_next_tuple parameter tbmres @ 2024-02-12 23:13 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Melanie Plageman @ 2024-02-12 23:13 UTC (permalink / raw) With the addition of the proposed streaming read API [1], table_scan_bitmap_next_block() will no longer take a TBMIterateResult as an input. Instead table AMs will be responsible for implementing a callback for the streaming read API which specifies which blocks should be prefetched and read. Thus, it no longer makes sense to use the TBMIterateResult as a means of communication between table_scan_bitmap_next_tuple() and table_scan_bitmap_next_block(). Note that this parameter was unused by heap AM's implementation of table_scan_bitmap_next_tuple(). [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam_handler.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 2 +- src/include/access/tableam.h | 12 +----------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 10c1c3b616b..a1ec50ab7a8 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -2248,7 +2248,6 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, - TBMIterateResult *tbmres, TupleTableSlot *slot) { HeapScanDesc hscan = (HeapScanDesc) scan; diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 94a5e2da17c..bbdaa591891 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -286,7 +286,7 @@ BitmapHeapNext(BitmapHeapScanState *node) /* * Attempt to fetch tuple from AM. */ - if (!table_scan_bitmap_next_tuple(scan, tbmres, slot)) + if (!table_scan_bitmap_next_tuple(scan, slot)) { /* nothing more to look at on this page */ node->tbmres = tbmres = NULL; diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index f1d0d4b78e3..e35bd36e710 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -787,10 +787,7 @@ typedef struct TableAmRoutine * * This will typically read and pin the target block, and do the necessary * work to allow scan_bitmap_next_tuple() to return tuples (e.g. it might - * make sense to perform tuple visibility checks at this time). For some - * AMs it will make more sense to do all the work referencing `tbmres` - * contents here, for others it might be better to defer more work to - * scan_bitmap_next_tuple. + * make sense to perform tuple visibility checks at this time). * * If `tbmres->blockno` is -1, this is a lossy scan and all visible tuples * on the page have to be returned, otherwise the tuples at offsets in @@ -821,15 +818,10 @@ typedef struct TableAmRoutine * Fetch the next tuple of a bitmap table scan into `slot` and return true * if a visible tuple was found, false otherwise. * - * For some AMs it will make more sense to do all the work referencing - * `tbmres` contents in scan_bitmap_next_block, for others it might be - * better to defer more work to this callback. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ bool (*scan_bitmap_next_tuple) (TableScanDesc scan, - struct TBMIterateResult *tbmres, TupleTableSlot *slot); /* @@ -1988,7 +1980,6 @@ table_scan_bitmap_next_block(TableScanDesc scan, */ static inline bool table_scan_bitmap_next_tuple(TableScanDesc scan, - struct TBMIterateResult *tbmres, TupleTableSlot *slot) { /* @@ -2000,7 +1991,6 @@ table_scan_bitmap_next_tuple(TableScanDesc scan, elog(ERROR, "unexpected table_scan_bitmap_next_tuple call during logical decoding"); return scan->rs_rd->rd_tableam->scan_bitmap_next_tuple(scan, - tbmres, slot); } -- 2.40.1 --kqqpqghcwbcc3dt5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0009-Make-table_scan_bitmap_next_block-async-friendly.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
end of thread, other threads:[~2024-02-12 23:13 UTC | newest] Thread overview: 21+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-06-21 21:00 [PATCH 7/7] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-06-21 21:00 [PATCH 7/7] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-07-31 12:06 [PATCH v5 3/3] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-07-31 12:06 [PATCH v4 3/3] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-09-16 08:07 [PATCH v6 3/3] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-09-16 08:07 [PATCH v8 4/4] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-09-16 08:07 [PATCH v7 5/5] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2021-10-28 01:50 pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2022-04-11 06:48 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Thomas Munro <[email protected]> 2022-04-11 07:43 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2022-04-12 03:49 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2022-04-16 20:56 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Thomas Munro <[email protected]> 2022-04-17 04:17 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2022-04-17 14:56 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Andrew Dunstan <[email protected]> 2022-04-17 23:49 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2022-04-18 17:32 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Andrew Dunstan <[email protected]> 2022-04-18 21:45 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Thomas Munro <[email protected]> 2022-04-19 02:33 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2022-04-18 04:55 ` Re: pgsql: Add TAP test for archive_cleanup_command and recovery_end_comman Michael Paquier <[email protected]> 2021-12-17 07:00 [PATCH v9 5/5] Move code to apply one WAL record to a subroutine. Heikki Linnakangas <[email protected]> 2024-02-12 23:13 [PATCH v7 08/13] Remove table_scan_bitmap_next_tuple parameter tbmres Melanie Plageman <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox