From: Alvaro Herrera <alvherre@alvh.no-ip.org>
To: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Cc: bossartn@amazon.com
Cc: andres@anarazel.de
Cc: masao.fujii@oss.nttdata.com
Cc: pgsql-hackers@lists.postgresql.org
Cc: mengjuan.cmj@alibaba-inc.com
Cc: Jakub.Wartak@tomtom.com
Subject: Re: prevent immature WAL streaming
Date: Fri, 17 Sep 2021 13:37:05 -0300
Message-ID: <202109171637.uhx5nnwef2yu@alvherre.pgsql> (raw)
In-Reply-To: <20210915.120056.345065921023162856.horikyota.ntt@gmail.com>
OK, this version is much more palatable, because here we verify that the
OVERWRITE_CONTRECORD we replay matches the record that was lost. Also,
I wrote a test script that creates such a broken record (by the simple
expedient of deleting the WAL file containing the second half while the
server is down); we then create a standby and we can observe that it
replays the sequence correctly.
If you have some time to try your reproducers with this new proposed
fix, I would appreciate it.
Added Matsumura-san to CC, because he was interested in this topic too
per the earlier thread.
--
Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/Attachments:
[text/x-diff] v7-0001-Implement-FIRST_IS_ABORTED_CONTRECORD.patch (17.9K, ../202109171637.uhx5nnwef2yu@alvherre.pgsql/2-v7-0001-Implement-FIRST_IS_ABORTED_CONTRECORD.patch)
download | inline diff:
From 0404d47ca1456771a1be4c8afb2c1bd1e0fa0f08 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Thu, 2 Sep 2021 17:21:46 -0400
Subject: [PATCH v7 1/2] Implement FIRST_IS_ABORTED_CONTRECORD
---
src/backend/access/rmgrdesc/xlogdesc.c | 11 ++
src/backend/access/transam/xlog.c | 153 ++++++++++++++++++++++--
src/backend/access/transam/xloginsert.c | 1 +
src/backend/access/transam/xlogreader.c | 54 ++++++++-
src/include/access/xlog_internal.h | 20 +++-
src/include/access/xlogreader.h | 14 +++
src/include/catalog/pg_control.h | 1 +
7 files changed, 244 insertions(+), 10 deletions(-)
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.cindex e6090a9dad..0be382f8a5 100644--- a/src/backend/access/rmgrdesc/xlogdesc.c+++ b/src/backend/access/rmgrdesc/xlogdesc.c@@ -139,6 +139,14 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
xlrec.ThisTimeLineID, xlrec.PrevTimeLineID,
timestamptz_to_str(xlrec.end_time));
}
+ else if (info == XLOG_OVERWRITE_CONTRECORD)+ {+ xl_overwrite_contrecord xlrec;++ memcpy(&xlrec, rec, sizeof(xl_overwrite_contrecord));+ appendStringInfo(buf, "lsn %X/%X",+ LSN_FORMAT_ARGS(xlrec.overwritten_lsn));+ }
}
const char *
@@ -178,6 +186,9 @@ xlog_identify(uint8 info)
case XLOG_END_OF_RECOVERY:
id = "END_OF_RECOVERY";
break;
+ case XLOG_OVERWRITE_CONTRECORD:+ id = "OVERWRITE_CONTRECORD";+ break;
case XLOG_FPI:
id = "FPI";
break;
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.cindex e51a7a749d..b6af3eb258 100644--- a/src/backend/access/transam/xlog.c+++ b/src/backend/access/transam/xlog.c@@ -199,6 +199,15 @@ static XLogRecPtr LastRec;
static XLogRecPtr flushedUpto = 0;
static TimeLineID receiveTLI = 0;
+/*+ * abortedRecPtr is the start pointer of a broken record at end of WAL when+ * recovery completes; missingContrecPtr is the location of the first+ * contrecord that went missing. See CreateOverwriteContrecordRecord for+ * details.+ */+static XLogRecPtr abortedRecPtr;+static XLogRecPtr missingContrecPtr;+
/*
* During recovery, lastFullPageWrites keeps track of full_page_writes that
* the replayed WAL records indicate. It's initialized with full_page_writes
@@ -894,6 +903,7 @@ static void checkTimeLineSwitch(XLogRecPtr lsn, TimeLineID newTLI,
TimeLineID prevTLI);
static void LocalSetXLogInsertAllowed(void);
static void CreateEndOfRecoveryRecord(void);
+static XLogRecPtr CreateOverwriteContrecordRecord(XLogRecPtr aborted_lsn);
static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
@@ -927,6 +937,7 @@ static XLogRecord *ReadRecord(XLogReaderState *xlogreader,
static void CheckRecoveryConsistency(void);
static XLogRecord *ReadCheckpointRecord(XLogReaderState *xlogreader,
XLogRecPtr RecPtr, int whichChkpt, bool report);
+static void VerifyOverwrittenRecord(XLogReaderState *record);
static bool rescanLatestTimeLine(void);
static void InitControlFile(uint64 sysidentifier);
static void WriteControlFile(void);
@@ -1120,8 +1131,8 @@ XLogInsertRecord(XLogRecData *rdata,
* All the record data, including the header, is now ready to be
* inserted. Copy the record in the space reserved.
*/
- CopyXLogRecordToWAL(rechdr->xl_tot_len, isLogSwitch, rdata,- StartPos, EndPos);+ CopyXLogRecordToWAL(rechdr->xl_tot_len, isLogSwitch,+ rdata, StartPos, EndPos);
/*
* Unless record is flagged as not important, update LSN of last
@@ -1504,7 +1515,8 @@ checkXLogConsistency(XLogReaderState *record)
* area in the WAL.
*/
static void
-CopyXLogRecordToWAL(int write_len, bool isLogSwitch, XLogRecData *rdata,+CopyXLogRecordToWAL(int write_len, bool isLogSwitch,+ XLogRecData *rdata,
XLogRecPtr StartPos, XLogRecPtr EndPos)
{
char *currpos;
@@ -2246,6 +2258,19 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, bool opportunistic)
if (!Insert->forcePageWrites)
NewPage->xlp_info |= XLP_BKP_REMOVABLE;
+ /*+ * If a record was found to be broken at the end of recovery, and+ * we're going to write on the page where its first contrecord was+ * lost, set the XLP_FIRST_IS_ABORTED_PARTIAL flag on the page header.+ * This instructs readers to skip the broken record that precedes+ * this.+ */+ if (missingContrecPtr == NewPageBeginPtr)+ {+ NewPage->xlp_info |= XLP_FIRST_IS_ABORTED_PARTIAL;+ missingContrecPtr = InvalidXLogRecPtr;+ }+
/*
* If first page of an XLOG segment file, make it a long header.
*/
@@ -4394,6 +4419,17 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
EndRecPtr = xlogreader->EndRecPtr;
if (record == NULL)
{
+ /*+ * When WAL ends in an incomplete record, keep track of it. After+ * recovery is done, we'll write a record to indicate downstream+ * WAL readers that that portion is to be ignored.+ */+ if (!XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))+ {+ abortedRecPtr = xlogreader->abortedRecPtr;+ missingContrecPtr = xlogreader->missingContrecPtr;+ }+
if (readFile >= 0)
{
close(readFile);
@@ -4516,6 +4552,36 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
}
}
+/*+ * When we find an XLOG_OVERWRITE_CONTRECORD record, verify that it matches+ * what xlogreader says it had read prior to throwing it all away.+ */+static void+VerifyOverwrittenRecord(XLogReaderState *record)+{+ xl_overwrite_contrecord xlrec;++ if (XLogRecGetRmid(record) != RM_XLOG_ID)+ elog(FATAL, "not XLOG rmgr");++ if ((XLogRecGetInfo(record) & ~XLR_INFO_MASK) != XLOG_OVERWRITE_CONTRECORD)+ elog(FATAL, "not OVERWRITE_CONTRECORD");++ memcpy(&xlrec, XLogRecGetData(record), sizeof(xl_overwrite_contrecord));++ if (xlrec.overwritten_lsn != record->overwrittenRecPtr)+ elog(FATAL, "mismatching overwritten LSN %X/%X -> %X/%X",+ LSN_FORMAT_ARGS(xlrec.overwritten_lsn),+ LSN_FORMAT_ARGS(record->overwrittenRecPtr));++ ereport(LOG,+ (errmsg("sucessfully skipped missing contrecord at %X/%X",+ LSN_FORMAT_ARGS(xlrec.overwritten_lsn))));++ /* Verifying the record should only happen once */+ record->overwrittenRecPtr = InvalidXLogRecPtr;+}+
/*
* Scan for new timelines that might have appeared in the archive since we
* started recovery.
@@ -7069,6 +7135,12 @@ StartupXLOG(void)
InRecovery = true;
}
+ /*+ * Start recovery assuming that the final record isn't lost.+ */+ abortedRecPtr = InvalidXLogRecPtr;+ missingContrecPtr = InvalidXLogRecPtr;+
/* REDO */
if (InRecovery)
{
@@ -7655,8 +7727,9 @@ StartupXLOG(void)
/*
* Kill WAL receiver, if it's still running, before we continue to write
- * the startup checkpoint record. It will trump over the checkpoint and- * subsequent records if it's still alive when we start writing WAL.+ * the startup checkpoint and aborted-contrecord records. It will trump+ * over these records and subsequent ones if it's still alive when we+ * start writing WAL.
*/
XLogShutdownWalRcv();
@@ -7689,8 +7762,12 @@ StartupXLOG(void)
StandbyMode = false;
/*
- * Re-fetch the last valid or last applied record, so we can identify the- * exact endpoint of what we consider the valid portion of WAL.+ * Determine where to start writing WAL next.+ *+ * When recovery ended in an incomplete record, write a WAL record about+ * that and continue after it. In all other cases, re-fetch the last+ * valid or last applied record, so we can identify the exact endpoint of+ * what we consider the valid portion of WAL.
*/
XLogBeginRead(xlogreader, LastRec);
record = ReadRecord(xlogreader, PANIC, false);
@@ -7821,6 +7898,18 @@ StartupXLOG(void)
XLogCtl->ThisTimeLineID = ThisTimeLineID;
XLogCtl->PrevTimeLineID = PrevTimeLineID;
+ /*+ * Actually, if WAL ended in an incomplete record, skip the parts that+ * made it through and start writing after the portion that persisted.+ * (It's critical to first write an OVERWRITE_CONTRECORD message, which+ * we'll do as soon as we're open for writing new WAL.)+ */+ if (!XLogRecPtrIsInvalid(missingContrecPtr))+ {+ Assert(!XLogRecPtrIsInvalid(abortedRecPtr));+ EndOfLog = missingContrecPtr;+ }+
/*
* Prepare to write WAL starting at EndOfLog location, and init xlog
* buffer cache using the block containing the last record from the
@@ -7873,13 +7962,23 @@ StartupXLOG(void)
XLogCtl->LogwrtRqst.Write = EndOfLog;
XLogCtl->LogwrtRqst.Flush = EndOfLog;
+ LocalSetXLogInsertAllowed();++ /* If necessary, write overwrite-contrecord before doing anything else */+ if (!XLogRecPtrIsInvalid(abortedRecPtr))+ {+ Assert(!XLogRecPtrIsInvalid(missingContrecPtr));+ CreateOverwriteContrecordRecord(abortedRecPtr);+ abortedRecPtr = InvalidXLogRecPtr;+ missingContrecPtr = InvalidXLogRecPtr;+ }+
/*
* Update full_page_writes in shared memory and write an XLOG_FPW_CHANGE
* record before resource manager writes cleanup WAL records or checkpoint
* record is written.
*/
Insert->fullPageWrites = lastFullPageWrites;
- LocalSetXLogInsertAllowed();
UpdateFullPageWrites();
LocalXLogInsertAllowed = -1;
@@ -9365,6 +9464,40 @@ CreateEndOfRecoveryRecord(void)
LocalXLogInsertAllowed = -1; /* return to "check" state */
}
+/*+ * Write an OVERWRITE_CONTRECORD message, which is used after recovery is+ * complete to mark the spot where a partial message was lost. We must not+ * overwrite the preserved portions of the partial message, because+ * downstream WAL consumers (physical replicas) may have already received+ * those parts and would not retreat to overwrite them.+ */+static XLogRecPtr+CreateOverwriteContrecordRecord(XLogRecPtr aborted_lsn)+{+ xl_overwrite_contrecord xlrec;+ XLogRecPtr recptr;++ /* sanity check */+ if (!RecoveryInProgress())+ elog(ERROR, "can only be used at end of recovery");++ xlrec.overwritten_lsn = aborted_lsn;+ /* XXX include the CRC-so-far of the old message? */++ START_CRIT_SECTION();++ XLogBeginInsert();+ XLogRegisterData((char *) &xlrec, sizeof(xl_overwrite_contrecord));++ recptr = XLogInsert(RM_XLOG_ID, XLOG_OVERWRITE_CONTRECORD);++ XLogFlush(recptr);++ END_CRIT_SECTION();++ return recptr;+}+
/*
* Flush all data in shared memory to disk, and fsync
*
@@ -10295,6 +10428,10 @@ xlog_redo(XLogReaderState *record)
RecoveryRestartPoint(&checkPoint);
}
+ else if (info == XLOG_OVERWRITE_CONTRECORD)+ {+ VerifyOverwrittenRecord(record);+ }
else if (info == XLOG_END_OF_RECOVERY)
{
xl_end_of_recovery xlrec;
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.cindex e596a0470a..a33cb23ac0 100644--- a/src/backend/access/transam/xloginsert.c+++ b/src/backend/access/transam/xloginsert.c@@ -409,6 +409,7 @@ XLogRegisterBufData(uint8 block_id, char *data, int len)
* - XLOG_MARK_UNIMPORTANT, to signal that the record is not important for
* durability, which allows to avoid triggering WAL archiving and other
* background activity.
+ * - XLOG_INCLUDE_XID, ???
*/
void
XLogSetRecordFlags(uint8 flags)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.cindex 5cf74e181a..b72a8bf9f6 100644--- a/src/backend/access/transam/xlogreader.c+++ b/src/backend/access/transam/xlogreader.c@@ -39,6 +39,7 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
+static void RememberOverwrittenRecord(XLogReaderState *state);
static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen);
static void XLogReaderInvalReadState(XLogReaderState *state);
@@ -278,6 +279,7 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
total_len;
uint32 targetRecOff;
uint32 pageHeaderSize;
+ bool assembled;
bool gotheader;
int readOff;
@@ -293,6 +295,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
state->errormsg_buf[0] = '\0';
ResetDecoder(state);
+ state->abortedRecPtr = InvalidXLogRecPtr;+ state->missingContrecPtr = InvalidXLogRecPtr;
RecPtr = state->EndRecPtr;
@@ -319,7 +323,9 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
randAccess = true;
}
+restart:
state->currRecPtr = RecPtr;
+ assembled = false;
targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ);
targetRecOff = RecPtr % XLOG_BLCKSZ;
@@ -415,6 +421,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
char *buffer;
uint32 gotlen;
+ assembled = true;+
/*
* Enlarge readRecordBuf as needed.
*/
@@ -448,8 +456,24 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
Assert(SizeOfXLogShortPHD <= readOff);
- /* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
++ /*+ * If we were expecting a continuation record and got an "aborted+ * partial" flag, that means the continuation record was lost.+ * Ignore the record we were reading, since we now know it's broken+ * and lost forever, and restart the read by assuming the address+ * to read is the location where we found this flag.+ */+ if (pageHeader->xlp_info & XLP_FIRST_IS_ABORTED_PARTIAL)+ {+ RememberOverwrittenRecord(state);+ ResetDecoder(state);+ RecPtr = targetPagePtr;+ goto restart;+ }++ /* Check that the continuation on next page looks valid */
if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD))
{
report_invalid_record(state,
@@ -551,6 +575,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
return NULL;
err:
+ if (assembled)+ {+ /*+ * We get here when a record that spans multiple pages needs to be+ * assembled, but something went wrong -- perhaps a contrecord piece+ * was lost. If caller is WAL replay, it will know where the aborted+ * record was and where to direct followup WAL to be written, marking+ * the next piece with XLP_FIRST_IS_ABORTED_PARTIAL, which will in+ * turn signal downstream WAL consumers that the broken WAL record is+ * to be ignored.+ */+ state->abortedRecPtr = RecPtr;+ state->missingContrecPtr = targetPagePtr;+ }
/*
* Invalidate the read state. We might read from a different source after
@@ -564,6 +602,20 @@ err:
return NULL;
}
+/*+ * When we find XLP_FIRST_IS_ABORTED_PARTIAL and stop reading a record that+ * was incompletely reading, keep track of what it is that we're aborting+ * before throwing away the data we already read. This allows somebody else to+ * verify that the next record we're going to find (which should be a+ * XLOG_OVERWRITE_CONTRECORD record) matches what we throw away.+ */+static void+RememberOverwrittenRecord(XLogReaderState *state)+{+ state->overwrittenRecPtr = state->currRecPtr;+ /* XXX more? could store CRC-so-far, for example */+}+
/*
* Read a single xlog page including at least [pageptr, reqLen] of valid data
* via the page_read() callback.
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.hindex 3b5eceff65..0fb44ba37d 100644--- a/src/include/access/xlog_internal.h+++ b/src/include/access/xlog_internal.h@@ -76,8 +76,20 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
#define XLP_LONG_HEADER 0x0002
/* This flag indicates backup blocks starting in this page are optional */
#define XLP_BKP_REMOVABLE 0x0004
+/*+ * This flag marks a record that replaces a missing contrecord.+ * When on WAL replay we expect a continuation record at the start of+ * a page that is not there, recovery ends but the checkpoint record+ * that follows is marked with this flag, which indicates WAL readers+ * that the incomplete record is to be skipped, and that WAL reading+ * is to be resumed here. This is useful for downstream consumers of+ * WAL which have already received (the first half of) the original+ * broken WAL record, such as via archive_command or physical streaming+ * replication, which we cannot "rewind".+ */+#define XLP_FIRST_IS_ABORTED_PARTIAL 0x0008
/* All defined flag bits in xlp_info (used for validity checking of header) */
-#define XLP_ALL_FLAGS 0x0007+#define XLP_ALL_FLAGS 0x000F
#define XLogPageHeaderSize(hdr) \
(((hdr)->xlp_info & XLP_LONG_HEADER) ? SizeOfXLogLongPHD : SizeOfXLogShortPHD)
@@ -249,6 +261,12 @@ typedef struct xl_restore_point
char rp_name[MAXFNAMELEN];
} xl_restore_point;
+/* Overwrite of prior contrecord */+typedef struct xl_overwrite_contrecord+{+ XLogRecPtr overwritten_lsn;+} xl_overwrite_contrecord;+
/* End of recovery mark, when we don't do an END_OF_RECOVERY checkpoint */
typedef struct xl_end_of_recovery
{
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.hindex 21d200d3df..f75fcf9005 100644--- a/src/include/access/xlogreader.h+++ b/src/include/access/xlogreader.h@@ -144,6 +144,12 @@ typedef struct
uint16 data_bufsz;
} DecodedBkpBlock;
+/* Data about a record that was found incomplete */+typedef struct+{+ XLogRecPtr lsn;+} XLogOverwrittenRecord;+
struct XLogReaderState
{
/*
@@ -175,6 +181,14 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /*+ * The start point of a partial record at the end of WAL, and the start+ * location of its first contrecord that went missing.+ */+ XLogRecPtr abortedRecPtr;+ XLogRecPtr missingContrecPtr;+ /* Set when XLP_FIRST_IS_ABORTED_PARTIAL is found */+ XLogRecPtr overwrittenRecPtr;
/* ----------------------------------------
* Decoded representation of current record
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.hindex e3f48158ce..179efa9f4a 100644--- a/src/include/catalog/pg_control.h+++ b/src/include/catalog/pg_control.h@@ -76,6 +76,7 @@ typedef struct CheckPoint
#define XLOG_END_OF_RECOVERY 0x90
#define XLOG_FPI_FOR_HINT 0xA0
#define XLOG_FPI 0xB0
+#define XLOG_OVERWRITE_CONTRECORD 0xC0
/*
--
2.20.1
[text/x-diff] v7-0002-test-code.patch (3.7K, ../202109171637.uhx5nnwef2yu@alvherre.pgsql/3-v7-0002-test-code.patch)
download | inline diff:
From 86e4e9c85e2916f11ffb287c566d23652e7bd285 Mon Sep 17 00:00:00 2001
From: Alvaro Herrera <alvherre@alvh.no-ip.org>
Date: Wed, 15 Sep 2021 13:10:04 -0300
Subject: [PATCH v7 2/2] test code
---
src/test/recovery/t/026_aborted_contrecord.pl | 73 +++++++++++++++++++
1 file changed, 73 insertions(+)
create mode 100644 src/test/recovery/t/026_aborted_contrecord.pl
diff --git a/src/test/recovery/t/026_aborted_contrecord.pl b/src/test/recovery/t/026_aborted_contrecord.plnew file mode 100644index 0000000000..e13f324ae6--- /dev/null+++ b/src/test/recovery/t/026_aborted_contrecord.pl@@ -0,0 +1,73 @@+# Copyright (c) 2021, PostgreSQL Global Development Group++# Test that a WAL record that is partially archived can be recovered from.++use strict;+use warnings;++use PostgresNode;+use TestLib;+use Test::More;++plan tests => 3;++my $node = PostgresNode->new('primary');+$node->init(allows_streaming => 1);+$node->append_conf('postgresql.conf', 'wal_keep_size=1GB');+$node->append_conf('postgresql.conf', 'max_locks_per_transaction=1000');+$node->start;++$node->safe_psql('postgres', 'create table filler (a int)');+# First, measure how many bytes does the insertion of 1000 rows produce+my $start_lsn = $node->safe_psql('postgres', q{select pg_current_wal_insert_lsn() - '0/0'});+$node->safe_psql('postgres', 'insert into filler select * from generate_series(1, 1000)');+my $end_lsn = $node->safe_psql('postgres', q{select pg_current_wal_insert_lsn() - '0/0'});+my $rows_walsize = $end_lsn - $start_lsn;++# Now consume all remaining room in the current WAL segment, leaving+# space enough only for the start of a largish record.+$node->safe_psql('postgres', qq{+WITH setting AS (+ SELECT setting::int AS wal_segsize+ FROM pg_settings WHERE name = 'wal_segment_size'+)+INSERT INTO filler+SELECT g FROM setting,+ generate_series(1, 1000 * (wal_segsize - ((pg_current_wal_insert_lsn() - '0/0') % wal_segsize)) / $rows_walsize) g+});++my $initfile = $node->safe_psql('postgres', 'SELECT pg_walfile_name(pg_current_wal_insert_lsn())');+$node->safe_psql('postgres', qq{SELECT pg_logical_emit_message(true, 'test 026', repeat('xyzxz', 123456))});+#$node->safe_psql('postgres', qq{create table foo ()});+my $endfile = $node->safe_psql('postgres', 'SELECT pg_walfile_name(pg_current_wal_insert_lsn())');+ok($initfile != $endfile, "$initfile differs from $endfile");++# Now stop abruptly, to avoid a stop checkpoint. We can remove the tail file+# afterwards, and on startup the large message should be overwritten with new+# contents+$node->stop('immediate');++unlink $node->basedir . "/pgdata/pg_wal/$endfile" or die "could not unlink ".$node->basedir."/pgdata/pg_wal/$endfile: $!";++# OK, create a standby at this spot.+$node->backup_fs_cold('backup');+my $node_standby = PostgresNode->new('standby');+$node_standby->init_from_backup($node, 'backup', has_streaming => 1);++$node_standby->start;+$node->start;++$node->safe_psql('postgres', qq{create table foo (a text); insert into foo values ('hello')});+$node->safe_psql('postgres', qq{SELECT pg_logical_emit_message(true, 'test 026', 'AABBCC')});++my $until_lsn = $node->safe_psql('postgres', "SELECT pg_current_wal_lsn()");+my $caughtup_query = "SELECT '$until_lsn'::pg_lsn <= pg_last_wal_replay_lsn()";+$node_standby->poll_query_until('postgres', $caughtup_query)+ or die "Timed out while waiting for standby to catch up";++ok($node_standby->safe_psql('postgres', 'select * from foo') eq+ 'hello', 'standby replays past overwritten contrecord');++# Verify message appears in standby's log+my $log = slurp_file($node_standby->logfile);+like($log, qr[sucessfully skipped missing contrecord at], "found log line in standby");--
2.20.1
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: alvherre@alvh.no-ip.org, horikyota.ntt@gmail.com, bossartn@amazon.com, andres@anarazel.de, masao.fujii@oss.nttdata.com, pgsql-hackers@lists.postgresql.org, mengjuan.cmj@alibaba-inc.com, Jakub.Wartak@tomtom.com
Subject: Re: prevent immature WAL streaming
In-Reply-To: <202109171637.uhx5nnwef2yu@alvherre.pgsql>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox