[PATCH v49 3/7] Make archiver process an auxiliary process

public inbox for [email protected]  
help / color / mirror / Atom feed

[PATCH v49 3/7] Make archiver process an auxiliary process
55+ messages / 6 participants
[nested] [flat]

* [PATCH v49 3/7] Make archiver process an auxiliary process
@ 2020-03-13 07:59  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Kyotaro Horiguchi @ 2020-03-13 07:59 UTC (permalink / raw)

This is a preliminary patch for shared-memory based stats collector.

Archiver process must be a auxiliary process since it uses shared
memory after stats data was moved into shared-memory. Make the process
an auxiliary process in order to make it work.
---
 src/backend/access/transam/xlogarchive.c |   7 +-
 src/backend/bootstrap/bootstrap.c        |  22 +--
 src/backend/postmaster/pgarch.c          | 174 +++--------------------
 src/backend/postmaster/postmaster.c      |  88 ++++++------
 src/backend/storage/lmgr/proc.c          |   1 +
 src/include/miscadmin.h                  |   2 +
 src/include/postmaster/pgarch.h          |  10 +-
 src/include/storage/pmsignal.h           |   1 -
 src/include/storage/proc.h               |  10 +-
 9 files changed, 83 insertions(+), 232 deletions(-)

diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 1c5a4f8b5a..8b66a30931 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -29,7 +29,8 @@
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
-#include "storage/pmsignal.h"
+#include "storage/latch.h"
+#include "storage/proc.h"
 
 /*
  * Attempt to retrieve the specified file from off-line archival storage.
@@ -490,8 +491,8 @@ XLogArchiveNotify(const char *xlog)
 	}
 
 	/* Notify archiver that it's got something to do */
-	if (IsUnderPostmaster)
-		SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER);
+	if (IsUnderPostmaster && ProcGlobal->archiverLatch)
+		SetLatch(ProcGlobal->archiverLatch);
 }
 
 /*
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index 6f615e6622..41da0c5059 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -317,6 +317,9 @@ AuxiliaryProcessMain(int argc, char *argv[])
 		case StartupProcess:
 			MyBackendType = B_STARTUP;
 			break;
+		case ArchiverProcess:
+			MyBackendType = B_ARCHIVER;
+			break;
 		case BgWriterProcess:
 			MyBackendType = B_BG_WRITER;
 			break;
@@ -437,30 +440,29 @@ AuxiliaryProcessMain(int argc, char *argv[])
 			proc_exit(1);		/* should never return */
 
 		case StartupProcess:
-			/* don't set signals, startup process has its own agenda */
 			StartupProcessMain();
-			proc_exit(1);		/* should never return */
+			proc_exit(1);
+
+		case ArchiverProcess:
+			PgArchiverMain();
+			proc_exit(1);
 
 		case BgWriterProcess:
-			/* don't set signals, bgwriter has its own agenda */
 			BackgroundWriterMain();
-			proc_exit(1);		/* should never return */
+			proc_exit(1);
 
 		case CheckpointerProcess:
-			/* don't set signals, checkpointer has its own agenda */
 			CheckpointerMain();
-			proc_exit(1);		/* should never return */
+			proc_exit(1);
 
 		case WalWriterProcess:
-			/* don't set signals, walwriter has its own agenda */
 			InitXLOGAccess();
 			WalWriterMain();
-			proc_exit(1);		/* should never return */
+			proc_exit(1);
 
 		case WalReceiverProcess:
-			/* don't set signals, walreceiver has its own agenda */
 			WalReceiverMain();
-			proc_exit(1);		/* should never return */
+			proc_exit(1);
 
 		default:
 			elog(PANIC, "unrecognized process type: %d", (int) MyAuxProcType);
diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c
index edec311f12..e237cedaff 100644
--- a/src/backend/postmaster/pgarch.c
+++ b/src/backend/postmaster/pgarch.c
@@ -48,6 +48,7 @@
 #include "storage/latch.h"
 #include "storage/pg_shmem.h"
 #include "storage/pmsignal.h"
+#include "storage/procsignal.h"
 #include "utils/guc.h"
 #include "utils/ps_status.h"
 
@@ -78,25 +79,17 @@
  * Local data
  * ----------
  */
-static time_t last_pgarch_start_time;
 static time_t last_sigterm_time = 0;
 
 /*
  * Flags set by interrupt handlers for later service in the main loop.
  */
-static volatile sig_atomic_t wakened = false;
 static volatile sig_atomic_t ready_to_stop = false;
 
 /* ----------
  * Local function forward declarations
  * ----------
  */
-#ifdef EXEC_BACKEND
-static pid_t pgarch_forkexec(void);
-#endif
-
-NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn();
-static void pgarch_waken(SIGNAL_ARGS);
 static void pgarch_waken_stop(SIGNAL_ARGS);
 static void pgarch_MainLoop(void);
 static void pgarch_ArchiverCopyLoop(void);
@@ -105,121 +98,9 @@ static bool pgarch_readyXlog(char *xlog);
 static void pgarch_archiveDone(char *xlog);
 
 
-/* ------------------------------------------------------------
- * Public functions called from postmaster follow
- * ------------------------------------------------------------
- */
-
-/*
- * pgarch_start
- *
- *	Called from postmaster at startup or after an existing archiver
- *	died.  Attempt to fire up a fresh archiver process.
- *
- *	Returns PID of child process, or 0 if fail.
- *
- *	Note: if fail, we will be called again from the postmaster main loop.
- */
-int
-pgarch_start(void)
-{
-	time_t		curtime;
-	pid_t		pgArchPid;
-
-	/*
-	 * Do nothing if no archiver needed
-	 */
-	if (!XLogArchivingActive())
-		return 0;
-
-	/*
-	 * Do nothing if too soon since last archiver start.  This is a safety
-	 * valve to protect against continuous respawn attempts if the archiver is
-	 * dying immediately at launch. Note that since we will be re-called from
-	 * the postmaster main loop, we will get another chance later.
-	 */
-	curtime = time(NULL);
-	if ((unsigned int) (curtime - last_pgarch_start_time) <
-		(unsigned int) PGARCH_RESTART_INTERVAL)
-		return 0;
-	last_pgarch_start_time = curtime;
-
-#ifdef EXEC_BACKEND
-	switch ((pgArchPid = pgarch_forkexec()))
-#else
-	switch ((pgArchPid = fork_process()))
-#endif
-	{
-		case -1:
-			ereport(LOG,
-					(errmsg("could not fork archiver: %m")));
-			return 0;
-
-#ifndef EXEC_BACKEND
-		case 0:
-			/* in postmaster child ... */
-			InitPostmasterChild();
-
-			/* Close the postmaster's sockets */
-			ClosePostmasterPorts(false);
-
-			/* Drop our connection to postmaster's shared memory, as well */
-			dsm_detach_all();
-			PGSharedMemoryDetach();
-
-			PgArchiverMain(0, NULL);
-			break;
-#endif
-
-		default:
-			return (int) pgArchPid;
-	}
-
-	/* shouldn't get here */
-	return 0;
-}
-
-/* ------------------------------------------------------------
- * Local functions called by archiver follow
- * ------------------------------------------------------------
- */
-
-
-#ifdef EXEC_BACKEND
-
-/*
- * pgarch_forkexec() -
- *
- * Format up the arglist for, then fork and exec, archive process
- */
-static pid_t
-pgarch_forkexec(void)
-{
-	char	   *av[10];
-	int			ac = 0;
-
-	av[ac++] = "postgres";
-
-	av[ac++] = "--forkarch";
-
-	av[ac++] = NULL;			/* filled in by postmaster_forkexec */
-
-	av[ac] = NULL;
-	Assert(ac < lengthof(av));
-
-	return postmaster_forkexec(ac, av);
-}
-#endif							/* EXEC_BACKEND */
-
-
-/*
- * PgArchiverMain
- *
- *	The argc/argv parameters are valid only in EXEC_BACKEND case.  However,
- *	since we don't use 'em, it hardly matters...
- */
-NON_EXEC_STATIC void
-PgArchiverMain(int argc, char *argv[])
+/* Main entry point for archiver process */
+void
+PgArchiverMain(void)
 {
 	/*
 	 * Ignore all signals usually bound to some action in the postmaster,
@@ -231,31 +112,24 @@ PgArchiverMain(int argc, char *argv[])
 	/* SIGQUIT handler was already set up by InitPostmasterChild */
 	pqsignal(SIGALRM, SIG_IGN);
 	pqsignal(SIGPIPE, SIG_IGN);
-	pqsignal(SIGUSR1, pgarch_waken);
+	pqsignal(SIGUSR1, procsignal_sigusr1_handler);
 	pqsignal(SIGUSR2, pgarch_waken_stop);
+
 	/* Reset some signals that are accepted by postmaster but not here */
 	pqsignal(SIGCHLD, SIG_DFL);
+
+	/* Unblock signals (they were blocked when the postmaster forked us) */
 	PG_SETMASK(&UnBlockSig);
 
-	MyBackendType = B_ARCHIVER;
-	init_ps_display(NULL);
+	/*
+	 * Advertise our latch that backends can use to wake us up while we're
+	 * sleeping.
+	 */
+	ProcGlobal->archiverLatch = &MyProc->procLatch;
 
 	pgarch_MainLoop();
 
-	exit(0);
-}
-
-/* SIGUSR1 signal handler for archiver process */
-static void
-pgarch_waken(SIGNAL_ARGS)
-{
-	int			save_errno = errno;
-
-	/* set flag that there is work to be done */
-	wakened = true;
-	SetLatch(MyLatch);
-
-	errno = save_errno;
+	proc_exit(0);
 }
 
 /* SIGUSR2 signal handler for archiver process */
@@ -282,14 +156,6 @@ pgarch_MainLoop(void)
 	pg_time_t	last_copy_time = 0;
 	bool		time_to_stop;
 
-	/*
-	 * We run the copy loop immediately upon entry, in case there are
-	 * unarchived files left over from a previous database run (or maybe the
-	 * archiver died unexpectedly).  After that we wait for a signal or
-	 * timeout before doing more.
-	 */
-	wakened = true;
-
 	/*
 	 * There shouldn't be anything for the archiver to do except to wait for a
 	 * signal ... however, the archiver exists to protect our data, so she
@@ -328,12 +194,8 @@ pgarch_MainLoop(void)
 		}
 
 		/* Do what we're here for */
-		if (wakened || time_to_stop)
-		{
-			wakened = false;
-			pgarch_ArchiverCopyLoop();
-			last_copy_time = time(NULL);
-		}
+		pgarch_ArchiverCopyLoop();
+		last_copy_time = time(NULL);
 
 		/*
 		 * Sleep until a signal is received, or until a poll is forced by
@@ -354,13 +216,9 @@ pgarch_MainLoop(void)
 							   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
 							   timeout * 1000L,
 							   WAIT_EVENT_ARCHIVER_MAIN);
-				if (rc & WL_TIMEOUT)
-					wakened = true;
 				if (rc & WL_POSTMASTER_DEATH)
 					time_to_stop = true;
 			}
-			else
-				wakened = true;
 		}
 
 		/*
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index edab95a19e..d9f8d82650 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -548,6 +548,7 @@ static void ShmemBackendArrayRemove(Backend *bn);
 #endif							/* EXEC_BACKEND */
 
 #define StartupDataBase()		StartChildProcess(StartupProcess)
+#define StartArchiver()			StartChildProcess(ArchiverProcess)
 #define StartBackgroundWriter() StartChildProcess(BgWriterProcess)
 #define StartCheckpointer()		StartChildProcess(CheckpointerProcess)
 #define StartWalWriter()		StartChildProcess(WalWriterProcess)
@@ -1792,7 +1793,7 @@ ServerLoop(void)
 
 		/* If we have lost the archiver, try to start a new one. */
 		if (PgArchPID == 0 && PgArchStartupAllowed())
-			PgArchPID = pgarch_start();
+			PgArchPID = StartArchiver();
 
 		/* If we need to signal the autovacuum launcher, do so now */
 		if (avlauncher_needs_signal)
@@ -3007,7 +3008,7 @@ reaper(SIGNAL_ARGS)
 			if (!IsBinaryUpgrade && AutoVacuumingActive() && AutoVacPID == 0)
 				AutoVacPID = StartAutoVacLauncher();
 			if (PgArchStartupAllowed() && PgArchPID == 0)
-				PgArchPID = pgarch_start();
+				PgArchPID = StartArchiver();
 			if (PgStatPID == 0)
 				PgStatPID = pgstat_start();
 
@@ -3126,6 +3127,18 @@ reaper(SIGNAL_ARGS)
 			continue;
 		}
 
+		/*
+		 * Was it the archiver?  We treat it the same way to WAL receiver.
+		 */
+		if (pid == PgArchPID)
+		{
+			PgArchPID = 0;
+			if (!EXIT_STATUS_0(exitstatus) && !EXIT_STATUS_1(exitstatus))
+				HandleChildCrash(pid, exitstatus,
+								 _("archiver process"));
+			continue;
+		}
+
 		/*
 		 * Was it the autovacuum launcher?	Normal exit can be ignored; we'll
 		 * start a new one at the next iteration of the postmaster's main
@@ -3141,24 +3154,6 @@ reaper(SIGNAL_ARGS)
 			continue;
 		}
 
-		/*
-		 * Was it the archiver?  If so, just try to start a new one; no need
-		 * to force reset of the rest of the system.  (If fail, we'll try
-		 * again in future cycles of the main loop.).  Unless we were waiting
-		 * for it to shut down; don't restart it in that case, and
-		 * PostmasterStateMachine() will advance to the next shutdown step.
-		 */
-		if (pid == PgArchPID)
-		{
-			PgArchPID = 0;
-			if (!EXIT_STATUS_0(exitstatus))
-				LogChildExit(LOG, _("archiver process"),
-							 pid, exitstatus);
-			if (PgArchStartupAllowed())
-				PgArchPID = pgarch_start();
-			continue;
-		}
-
 		/*
 		 * Was it the statistics collector?  If so, just try to start a new
 		 * one; no need to force reset of the rest of the system.  (If fail,
@@ -3403,7 +3398,7 @@ CleanupBackend(int pid,
 
 /*
  * HandleChildCrash -- cleanup after failed backend, bgwriter, checkpointer,
- * walwriter, autovacuum, or background worker.
+ * walwriter, autovacuum, archiver or background worker.
  *
  * The objectives here are to clean up our local state about the child
  * process, and to signal all other remaining children to quickdie.
@@ -3609,19 +3604,16 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 		signal_child(AutoVacPID, (SendStop ? SIGSTOP : SIGQUIT));
 	}
 
-	/*
-	 * Force a power-cycle of the pgarch process too.  (This isn't absolutely
-	 * necessary, but it seems like a good idea for robustness, and it
-	 * simplifies the state-machine logic in the case where a shutdown request
-	 * arrives during crash processing.)
-	 */
-	if (PgArchPID != 0 && take_action)
+	/* Take care of the archiver too */
+	if (pid == PgArchPID)
+		PgArchPID = 0;
+	else if (PgArchPID != 0 && take_action)
 	{
 		ereport(DEBUG2,
 				(errmsg_internal("sending %s to process %d",
-								 "SIGQUIT",
+								 (SendStop ? "SIGSTOP" : "SIGQUIT"),
 								 (int) PgArchPID)));
-		signal_child(PgArchPID, SIGQUIT);
+		signal_child(PgArchPID, (SendStop ? SIGSTOP : SIGQUIT));
 	}
 
 	/*
@@ -3912,6 +3904,7 @@ PostmasterStateMachine(void)
 			Assert(CheckpointerPID == 0);
 			Assert(WalWriterPID == 0);
 			Assert(AutoVacPID == 0);
+			Assert(PgArchPID == 0);
 			/* syslogger is not considered here */
 			pmState = PM_NO_CHILDREN;
 		}
@@ -4989,6 +4982,19 @@ SubPostmasterMain(int argc, char *argv[])
 
 		AuxiliaryProcessMain(argc - 2, argv + 2);	/* does not return */
 	}
+	if (strcmp(argv[1], "--forkarch") == 0)
+	{
+		/* Restore basic shared memory pointers */
+		InitShmemAccess(UsedShmemSegAddr);
+
+		/* Need a PGPROC to run CreateSharedMemoryAndSemaphores */
+		InitAuxiliaryProcess();
+
+		/* Attach process to shared data structures */
+		CreateSharedMemoryAndSemaphores();
+
+		PgArchiverMain(); /* does not return */
+	}
 	if (strcmp(argv[1], "--forkavlauncher") == 0)
 	{
 		/* Restore basic shared memory pointers */
@@ -5037,12 +5043,6 @@ SubPostmasterMain(int argc, char *argv[])
 
 		StartBackgroundWorker();
 	}
-	if (strcmp(argv[1], "--forkarch") == 0)
-	{
-		/* Do not want to attach to shared memory */
-
-		PgArchiverMain(argc, argv); /* does not return */
-	}
 	if (strcmp(argv[1], "--forkcol") == 0)
 	{
 		/* Do not want to attach to shared memory */
@@ -5140,7 +5140,7 @@ sigusr1_handler(SIGNAL_ARGS)
 		 */
 		Assert(PgArchPID == 0);
 		if (XLogArchivingAlways())
-			PgArchPID = pgarch_start();
+			PgArchPID = StartArchiver();
 
 		/*
 		 * If we aren't planning to enter hot standby mode later, treat
@@ -5194,16 +5194,6 @@ sigusr1_handler(SIGNAL_ARGS)
 	if (StartWorkerNeeded || HaveCrashedWorker)
 		maybe_start_bgworkers();
 
-	if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER) &&
-		PgArchPID != 0)
-	{
-		/*
-		 * Send SIGUSR1 to archiver process, to wake it up and begin archiving
-		 * next WAL file.
-		 */
-		signal_child(PgArchPID, SIGUSR1);
-	}
-
 	/* Tell syslogger to rotate logfile if requested */
 	if (SysLoggerPID != 0)
 	{
@@ -5445,6 +5435,10 @@ StartChildProcess(AuxProcType type)
 				ereport(LOG,
 						(errmsg("could not fork startup process: %m")));
 				break;
+			case ArchiverProcess:
+				ereport(LOG,
+						(errmsg("could not fork archiver process: %m")));
+				break;
 			case BgWriterProcess:
 				ereport(LOG,
 						(errmsg("could not fork background writer process: %m")));
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 897045ee27..bf5d6da89b 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -182,6 +182,7 @@ InitProcGlobal(void)
 	ProcGlobal->startupBufferPinWaitBufId = -1;
 	ProcGlobal->walwriterLatch = NULL;
 	ProcGlobal->checkpointerLatch = NULL;
+	ProcGlobal->archiverLatch = NULL;
 	pg_atomic_init_u32(&ProcGlobal->procArrayGroupFirst, INVALID_PGPROCNO);
 	pg_atomic_init_u32(&ProcGlobal->clogGroupFirst, INVALID_PGPROCNO);
 
diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h
index 1bdc97e308..adb9f819bb 100644
--- a/src/include/miscadmin.h
+++ b/src/include/miscadmin.h
@@ -419,6 +419,7 @@ typedef enum
 	BootstrapProcess,
 	StartupProcess,
 	BgWriterProcess,
+	ArchiverProcess,
 	CheckpointerProcess,
 	WalWriterProcess,
 	WalReceiverProcess,
@@ -431,6 +432,7 @@ extern AuxProcType MyAuxProcType;
 #define AmBootstrapProcess()		(MyAuxProcType == BootstrapProcess)
 #define AmStartupProcess()			(MyAuxProcType == StartupProcess)
 #define AmBackgroundWriterProcess() (MyAuxProcType == BgWriterProcess)
+#define AmArchiverProcess()			(MyAuxProcType == ArchiverProcess)
 #define AmCheckpointerProcess()		(MyAuxProcType == CheckpointerProcess)
 #define AmWalWriterProcess()		(MyAuxProcType == WalWriterProcess)
 #define AmWalReceiverProcess()		(MyAuxProcType == WalReceiverProcess)
diff --git a/src/include/postmaster/pgarch.h b/src/include/postmaster/pgarch.h
index d102a21ab7..737ed69e57 100644
--- a/src/include/postmaster/pgarch.h
+++ b/src/include/postmaster/pgarch.h
@@ -26,14 +26,6 @@
 #define MAX_XFN_CHARS	40
 #define VALID_XFN_CHARS "0123456789ABCDEF.history.backup.partial"
 
-/* ----------
- * Functions called from postmaster
- * ----------
- */
-extern int	pgarch_start(void);
-
-#ifdef EXEC_BACKEND
-extern void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn();
-#endif
+extern void PgArchiverMain(void) pg_attribute_noreturn();
 
 #endif							/* _PGARCH_H */
diff --git a/src/include/storage/pmsignal.h b/src/include/storage/pmsignal.h
index dbbed18f61..8ed4d87ae6 100644
--- a/src/include/storage/pmsignal.h
+++ b/src/include/storage/pmsignal.h
@@ -34,7 +34,6 @@ typedef enum
 {
 	PMSIGNAL_RECOVERY_STARTED,	/* recovery has started */
 	PMSIGNAL_BEGIN_HOT_STANDBY, /* begin Hot Standby */
-	PMSIGNAL_WAKEN_ARCHIVER,	/* send a NOTIFY signal to xlog archiver */
 	PMSIGNAL_ROTATE_LOGFILE,	/* send SIGUSR1 to syslogger to rotate logfile */
 	PMSIGNAL_START_AUTOVAC_LAUNCHER,	/* start an autovacuum launcher */
 	PMSIGNAL_START_AUTOVAC_WORKER,	/* start an autovacuum worker */
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index a777cb64a1..d73890b228 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -357,6 +357,8 @@ typedef struct PROC_HDR
 	int			startupProcPid;
 	/* Buffer id of the buffer that Startup process waits for pin on, or -1 */
 	int			startupBufferPinWaitBufId;
+	/* Archiver process's latch */
+	Latch	   *archiverLatch;
 } PROC_HDR;
 
 extern PGDLLIMPORT PROC_HDR *ProcGlobal;
@@ -370,11 +372,11 @@ extern PGPROC *PreparedXactProcs;
  * We set aside some extra PGPROC structures for auxiliary processes,
  * ie things that aren't full-fledged backends but need shmem access.
  *
- * Background writer, checkpointer and WAL writer run during normal operation.
- * Startup process and WAL receiver also consume 2 slots, but WAL writer is
- * launched only after startup has exited, so we only need 4 slots.
+ * Background writer, checkpointer, WAL writer and archiver run during normal
+ * operation.  Startup process and WAL receiver also consume 2 slots, but WAL
+ * writer is launched only after startup has exited, so we only need 5 slots.
  */
-#define NUM_AUXILIARY_PROCS		4
+#define NUM_AUXILIARY_PROCS		5
 
 /* configurable options */
 extern PGDLLIMPORT int DeadlockTimeout;
-- 
2.27.0


----Next_Part(Tue_Mar__9_16_53_11_2021_575)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v49-0004-Shared-memory-based-stats-collector.patch"



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v3] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.
---
 .../t/035_standby_logical_decoding.pl         | 56 +++++++++----------
 1 file changed, 27 insertions(+), 29 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..9be1ff9b58 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,22 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_current();]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +264,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +485,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +562,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -588,13 +595,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +627,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
-- 
2.34.1


--s5+L671Fue1+NSGd--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v3] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.
---
 .../t/035_standby_logical_decoding.pl         | 56 +++++++++----------
 1 file changed, 27 insertions(+), 29 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..9be1ff9b58 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,22 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_current();]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +264,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +485,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +562,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -588,13 +595,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +627,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
-- 
2.34.1


--s5+L671Fue1+NSGd--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v6] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.
---
 .../t/035_standby_logical_decoding.pl         | 59 ++++++++++---------
 1 file changed, 30 insertions(+), 29 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..9bfa8833b5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,25 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +267,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +488,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +565,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -588,13 +598,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +630,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
-- 
2.34.1


--EA5NvN8PdJ4eq2qO--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v7] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

While at it, also fixing some typos/bad test description in it.
---
 .../t/035_standby_logical_decoding.pl         | 68 ++++++++++---------
 1 file changed, 35 insertions(+), 33 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..94406ad960 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,25 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +267,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +488,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +565,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -582,19 +592,15 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-# One way to produce recovery conflict is to create/drop a relation and
-# launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
+# One way to produce recovery conflict on a shared catalog table is to create/drop
+# a role and launch a vacuum on pg_authid with hot_standby_feedback turned off on
+# the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +631,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -671,7 +673,7 @@ $node_standby->restart;
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 4: conflict due to on-access pruning.
+# Scenario 5: conflict due to on-access pruning.
 ##################################################
 
 # get the position to search from in the standby logfile
@@ -711,7 +713,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 5: incorrect wal_level on primary.
+# Scenario 6: incorrect wal_level on primary.
 ##################################################
 
 # get the position to search from in the standby logfile
-- 
2.34.1


--UyWcF3I6fyAgeN0n--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v8] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

While at it, also fixing some typos/bad test description in it.
---
 .../t/035_standby_logical_decoding.pl         | 78 +++++++++++--------
 1 file changed, 45 insertions(+), 33 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..6664405772 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,35 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum. Arguments are: $vac_option (the option to be passed to the
+# vacuum command), $sql (the sql to launch before triggering the vacuum) and
+# $to_vac (the relation to vacuum).
+# Note that to get the horizon we're using pg_current_snapshot() (it does not
+# generate a Transaction/COMMIT wal record, so we don't increase the risk to see
+# a xl_running_xacts that would advance active replication slot's catalog_xmin).
+# Indeed advancing the active replication slot's catalog_xmin would break some
+# tests that expect the active slot to conflict with the catalog xmin horizon.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# Get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# Launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	# Wait until we get a newer horizon
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	# Launch the vacuum command and insert into flush_wal (see create table
+	# flush_wal for the reason why).
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +277,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +498,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +575,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -582,19 +602,15 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-# One way to produce recovery conflict is to create/drop a relation and
-# launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
+# One way to produce recovery conflict on a shared catalog table is to create/drop
+# a role and launch a vacuum on pg_authid with hot_standby_feedback turned off on
+# the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +641,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -671,7 +683,7 @@ $node_standby->restart;
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 4: conflict due to on-access pruning.
+# Scenario 5: conflict due to on-access pruning.
 ##################################################
 
 # get the position to search from in the standby logfile
@@ -711,7 +723,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 5: incorrect wal_level on primary.
+# Scenario 6: incorrect wal_level on primary.
 ##################################################
 
 # get the position to search from in the standby logfile
-- 
2.34.1


--aCojdxLXUGM6XwKA--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v3] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.
---
 .../t/035_standby_logical_decoding.pl         | 56 +++++++++----------
 1 file changed, 27 insertions(+), 29 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..9be1ff9b58 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,22 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_current();]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +264,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +485,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +562,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -588,13 +595,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +627,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
-- 
2.34.1


--s5+L671Fue1+NSGd--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v6] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.
---
 .../t/035_standby_logical_decoding.pl         | 59 ++++++++++---------
 1 file changed, 30 insertions(+), 29 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..9bfa8833b5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,25 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +267,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +488,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +565,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -588,13 +598,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +630,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
-- 
2.34.1


--EA5NvN8PdJ4eq2qO--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v7] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

While at it, also fixing some typos/bad test description in it.
---
 .../t/035_standby_logical_decoding.pl         | 68 ++++++++++---------
 1 file changed, 35 insertions(+), 33 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..94406ad960 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,25 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +267,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +488,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +565,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -582,19 +592,15 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-# One way to produce recovery conflict is to create/drop a relation and
-# launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
+# One way to produce recovery conflict on a shared catalog table is to create/drop
+# a role and launch a vacuum on pg_authid with hot_standby_feedback turned off on
+# the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +631,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -671,7 +673,7 @@ $node_standby->restart;
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 4: conflict due to on-access pruning.
+# Scenario 5: conflict due to on-access pruning.
 ##################################################
 
 # get the position to search from in the standby logfile
@@ -711,7 +713,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 5: incorrect wal_level on primary.
+# Scenario 6: incorrect wal_level on primary.
 ##################################################
 
 # get the position to search from in the standby logfile
-- 
2.34.1


--UyWcF3I6fyAgeN0n--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v8] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

While at it, also fixing some typos/bad test description in it.
---
 .../t/035_standby_logical_decoding.pl         | 78 +++++++++++--------
 1 file changed, 45 insertions(+), 33 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..6664405772 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,35 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum. Arguments are: $vac_option (the option to be passed to the
+# vacuum command), $sql (the sql to launch before triggering the vacuum) and
+# $to_vac (the relation to vacuum).
+# Note that to get the horizon we're using pg_current_snapshot() (it does not
+# generate a Transaction/COMMIT wal record, so we don't increase the risk to see
+# a xl_running_xacts that would advance active replication slot's catalog_xmin).
+# Indeed advancing the active replication slot's catalog_xmin would break some
+# tests that expect the active slot to conflict with the catalog xmin horizon.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# Get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# Launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	# Wait until we get a newer horizon
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	# Launch the vacuum command and insert into flush_wal (see create table
+	# flush_wal for the reason why).
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +277,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +498,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +575,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -582,19 +602,15 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-# One way to produce recovery conflict is to create/drop a relation and
-# launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
+# One way to produce recovery conflict on a shared catalog table is to create/drop
+# a role and launch a vacuum on pg_authid with hot_standby_feedback turned off on
+# the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +641,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -671,7 +683,7 @@ $node_standby->restart;
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 4: conflict due to on-access pruning.
+# Scenario 5: conflict due to on-access pruning.
 ##################################################
 
 # get the position to search from in the standby logfile
@@ -711,7 +723,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 5: incorrect wal_level on primary.
+# Scenario 6: incorrect wal_level on primary.
 ##################################################
 
 # get the position to search from in the standby logfile
-- 
2.34.1


--aCojdxLXUGM6XwKA--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v6] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.
---
 .../t/035_standby_logical_decoding.pl         | 59 ++++++++++---------
 1 file changed, 30 insertions(+), 29 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..9bfa8833b5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,25 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +267,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +488,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +565,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -588,13 +598,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +630,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
-- 
2.34.1


--EA5NvN8PdJ4eq2qO--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v7] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

While at it, also fixing some typos/bad test description in it.
---
 .../t/035_standby_logical_decoding.pl         | 68 ++++++++++---------
 1 file changed, 35 insertions(+), 33 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..94406ad960 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,25 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +267,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +488,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +565,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -582,19 +592,15 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-# One way to produce recovery conflict is to create/drop a relation and
-# launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
+# One way to produce recovery conflict on a shared catalog table is to create/drop
+# a role and launch a vacuum on pg_authid with hot_standby_feedback turned off on
+# the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +631,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -671,7 +673,7 @@ $node_standby->restart;
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 4: conflict due to on-access pruning.
+# Scenario 5: conflict due to on-access pruning.
 ##################################################
 
 # get the position to search from in the standby logfile
@@ -711,7 +713,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 5: incorrect wal_level on primary.
+# Scenario 6: incorrect wal_level on primary.
 ##################################################
 
 # get the position to search from in the standby logfile
-- 
2.34.1


--UyWcF3I6fyAgeN0n--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v8] Fix 035_standby_logical_decoding.pl race condition
@ 2024-01-09 05:08  bdrouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: bdrouvot @ 2024-01-09 05:08 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

While at it, also fixing some typos/bad test description in it.
---
 .../t/035_standby_logical_decoding.pl         | 78 +++++++++++--------
 1 file changed, 45 insertions(+), 33 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..6664405772 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -238,6 +238,35 @@ sub check_for_invalidation
 	) or die "Timed out waiting confl_active_logicalslot to be updated";
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum. Arguments are: $vac_option (the option to be passed to the
+# vacuum command), $sql (the sql to launch before triggering the vacuum) and
+# $to_vac (the relation to vacuum).
+# Note that to get the horizon we're using pg_current_snapshot() (it does not
+# generate a Transaction/COMMIT wal record, so we don't increase the risk to see
+# a xl_running_xacts that would advance active replication slot's catalog_xmin).
+# Indeed advancing the active replication slot's catalog_xmin would break some
+# tests that expect the active slot to conflict with the catalog xmin horizon.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	# Get the current xid horizon
+	my $xid_horizon = $node_primary->safe_psql('testdb', qq[select pg_snapshot_xmin(pg_current_snapshot());]);
+
+	# Launch our sql
+	$node_primary->safe_psql('testdb', qq[$sql]);
+
+	# Wait until we get a newer horizon
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	# Launch the vacuum command and insert into flush_wal (see create table
+	# flush_wal for the reason why).
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +277,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -468,13 +498,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -550,13 +575,8 @@ reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
 	0, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -582,19 +602,15 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-# One way to produce recovery conflict is to create/drop a relation and
-# launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
+# One way to produce recovery conflict on a shared catalog table is to create/drop
+# a role and launch a vacuum on pg_authid with hot_standby_feedback turned off on
+# the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
 	'shared_row_removal_', 0, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
@@ -625,14 +641,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -671,7 +683,7 @@ $node_standby->restart;
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 4: conflict due to on-access pruning.
+# Scenario 5: conflict due to on-access pruning.
 ##################################################
 
 # get the position to search from in the standby logfile
@@ -711,7 +723,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
-# Scenario 5: incorrect wal_level on primary.
+# Scenario 6: incorrect wal_level on primary.
 ##################################################
 
 # get the position to search from in the standby logfile
-- 
2.34.1


--aCojdxLXUGM6XwKA--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v4] Fix 035_standby_logical_decoding.pl race conditions
@ 2024-01-11 13:36  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2024-01-11 13:36 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

Also, get rid of the active slot invalidation check during the on-access pruning
check. This test is racy for active slots and active slots invalidations are well
covered in other tests.
---
 .../t/035_standby_logical_decoding.pl         | 114 +++++++++---------
 1 file changed, 59 insertions(+), 55 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..7a50187326 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -191,9 +191,9 @@ sub check_slots_conflict_reason
 
 # Drop the slots, re-create them, change hot_standby_feedback,
 # check xmin and catalog_xmin values, make slot active and reset stat.
-sub reactive_slots_change_hfs_and_wait_for_xmins
+sub recreate_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $activate) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
@@ -203,8 +203,11 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($activate)
+	{
+		$handle =
+			make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -213,7 +216,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $activated) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -230,14 +233,33 @@ sub check_for_invalidation
 		"activeslot slot invalidation is logged $test_name");
 
 	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($activated)
+	{
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_current();]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +270,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -464,22 +487,17 @@ $node_subscriber->stop;
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
@@ -546,22 +564,17 @@ my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -584,23 +597,18 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('row_removal_',
+	'shared_row_removal_', 0, 1, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -621,18 +629,14 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
+	'no_conflict_', 0, 1, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -679,8 +683,8 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+recreate_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -695,7 +699,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -739,7 +743,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify conflict_reason is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.34.1


--DdpMZ4HbwlFm/kjT
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v1-0001-Fix-race-condition-in-InvalidatePossiblyObsoleteS.patch"



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v5] Fix 035_standby_logical_decoding.pl race conditions
@ 2024-01-11 13:36  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2024-01-11 13:36 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

Also, get rid of the active slot invalidation check during the on-access pruning
check. This test is racy for active slots and active slots invalidations are well
covered in other tests.
---
 .../t/035_standby_logical_decoding.pl         | 114 +++++++++---------
 1 file changed, 59 insertions(+), 55 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..dd4149c8bc 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -191,9 +191,9 @@ sub check_slots_conflict_reason
 
 # Drop the slots, re-create them, change hot_standby_feedback,
 # check xmin and catalog_xmin values, make slot active and reset stat.
-sub reactive_slots_change_hfs_and_wait_for_xmins
+sub recreate_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $activate) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
@@ -203,8 +203,11 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($activate)
+	{
+		$handle =
+			make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -213,7 +216,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $activated) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -230,14 +233,33 @@ sub check_for_invalidation
 		"activeslot slot invalidation is logged $test_name");
 
 	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($activated)
+	{
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_snapshot_xmax(txid_current_snapshot()) - 1;]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +270,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -464,22 +487,17 @@ $node_subscriber->stop;
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
@@ -546,22 +564,17 @@ my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -584,23 +597,18 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('row_removal_',
+	'shared_row_removal_', 0, 1, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -621,18 +629,14 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
+	'no_conflict_', 0, 1, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -679,8 +683,8 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+recreate_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -695,7 +699,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -739,7 +743,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify conflict_reason is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.34.1


--y6m3n3Pgvo2G5ZzY--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v4] Fix 035_standby_logical_decoding.pl race conditions
@ 2024-01-11 13:36  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2024-01-11 13:36 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

Also, get rid of the active slot invalidation check during the on-access pruning
check. This test is racy for active slots and active slots invalidations are well
covered in other tests.
---
 .../t/035_standby_logical_decoding.pl         | 114 +++++++++---------
 1 file changed, 59 insertions(+), 55 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..7a50187326 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -191,9 +191,9 @@ sub check_slots_conflict_reason
 
 # Drop the slots, re-create them, change hot_standby_feedback,
 # check xmin and catalog_xmin values, make slot active and reset stat.
-sub reactive_slots_change_hfs_and_wait_for_xmins
+sub recreate_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $activate) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
@@ -203,8 +203,11 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($activate)
+	{
+		$handle =
+			make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -213,7 +216,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $activated) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -230,14 +233,33 @@ sub check_for_invalidation
 		"activeslot slot invalidation is logged $test_name");
 
 	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($activated)
+	{
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_current();]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +270,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -464,22 +487,17 @@ $node_subscriber->stop;
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
@@ -546,22 +564,17 @@ my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -584,23 +597,18 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('row_removal_',
+	'shared_row_removal_', 0, 1, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -621,18 +629,14 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
+	'no_conflict_', 0, 1, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -679,8 +683,8 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+recreate_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -695,7 +699,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -739,7 +743,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify conflict_reason is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.34.1


--DdpMZ4HbwlFm/kjT
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v1-0001-Fix-race-condition-in-InvalidatePossiblyObsoleteS.patch"



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v5] Fix 035_standby_logical_decoding.pl race conditions
@ 2024-01-11 13:36  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2024-01-11 13:36 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

Also, get rid of the active slot invalidation check during the on-access pruning
check. This test is racy for active slots and active slots invalidations are well
covered in other tests.
---
 .../t/035_standby_logical_decoding.pl         | 114 +++++++++---------
 1 file changed, 59 insertions(+), 55 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..dd4149c8bc 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -191,9 +191,9 @@ sub check_slots_conflict_reason
 
 # Drop the slots, re-create them, change hot_standby_feedback,
 # check xmin and catalog_xmin values, make slot active and reset stat.
-sub reactive_slots_change_hfs_and_wait_for_xmins
+sub recreate_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $activate) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
@@ -203,8 +203,11 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($activate)
+	{
+		$handle =
+			make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -213,7 +216,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $activated) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -230,14 +233,33 @@ sub check_for_invalidation
 		"activeslot slot invalidation is logged $test_name");
 
 	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($activated)
+	{
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_snapshot_xmax(txid_current_snapshot()) - 1;]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +270,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -464,22 +487,17 @@ $node_subscriber->stop;
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
@@ -546,22 +564,17 @@ my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -584,23 +597,18 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('row_removal_',
+	'shared_row_removal_', 0, 1, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -621,18 +629,14 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
+	'no_conflict_', 0, 1, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -679,8 +683,8 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+recreate_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -695,7 +699,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -739,7 +743,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify conflict_reason is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.34.1


--y6m3n3Pgvo2G5ZzY--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v4] Fix 035_standby_logical_decoding.pl race conditions
@ 2024-01-11 13:36  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2024-01-11 13:36 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

Also, get rid of the active slot invalidation check during the on-access pruning
check. This test is racy for active slots and active slots invalidations are well
covered in other tests.
---
 .../t/035_standby_logical_decoding.pl         | 114 +++++++++---------
 1 file changed, 59 insertions(+), 55 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..7a50187326 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -191,9 +191,9 @@ sub check_slots_conflict_reason
 
 # Drop the slots, re-create them, change hot_standby_feedback,
 # check xmin and catalog_xmin values, make slot active and reset stat.
-sub reactive_slots_change_hfs_and_wait_for_xmins
+sub recreate_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $activate) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
@@ -203,8 +203,11 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($activate)
+	{
+		$handle =
+			make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -213,7 +216,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $activated) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -230,14 +233,33 @@ sub check_for_invalidation
 		"activeslot slot invalidation is logged $test_name");
 
 	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($activated)
+	{
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_current();]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +270,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -464,22 +487,17 @@ $node_subscriber->stop;
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
@@ -546,22 +564,17 @@ my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -584,23 +597,18 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('row_removal_',
+	'shared_row_removal_', 0, 1, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -621,18 +629,14 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
+	'no_conflict_', 0, 1, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -679,8 +683,8 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+recreate_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -695,7 +699,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -739,7 +743,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify conflict_reason is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.34.1


--DdpMZ4HbwlFm/kjT
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v1-0001-Fix-race-condition-in-InvalidatePossiblyObsoleteS.patch"



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* [PATCH v5] Fix 035_standby_logical_decoding.pl race conditions
@ 2024-01-11 13:36  Bertrand Drouvot <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2024-01-11 13:36 UTC (permalink / raw)

We want to ensure that vacuum was able to remove dead rows (aka no other
transactions holding back global xmin) before testing for slots invalidation on
the standby.

Also, get rid of the active slot invalidation check during the on-access pruning
check. This test is racy for active slots and active slots invalidations are well
covered in other tests.
---
 .../t/035_standby_logical_decoding.pl         | 114 +++++++++---------
 1 file changed, 59 insertions(+), 55 deletions(-)
 100.0% src/test/recovery/t/

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8bc39a5f03..dd4149c8bc 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -191,9 +191,9 @@ sub check_slots_conflict_reason
 
 # Drop the slots, re-create them, change hot_standby_feedback,
 # check xmin and catalog_xmin values, make slot active and reset stat.
-sub reactive_slots_change_hfs_and_wait_for_xmins
+sub recreate_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $activate) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
@@ -203,8 +203,11 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($activate)
+	{
+		$handle =
+			make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -213,7 +216,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $activated) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -230,14 +233,33 @@ sub check_for_invalidation
 		"activeslot slot invalidation is logged $test_name");
 
 	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($activated)
+	{
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
+# Launch $sql and wait for a new snapshot that has a newer horizon before
+# doing the vacuum with $vac_option on $to_vac.
+sub wait_until_vacuum_can_remove
+{
+	my ($vac_option, $sql, $to_vac) = @_;
+
+	my $xid = $node_primary->safe_psql('testdb', qq[$sql
+													select txid_snapshot_xmax(txid_current_snapshot()) - 1;]);
+
+	$node_primary->poll_query_until('testdb',
+		"SELECT (select txid_snapshot_xmin(txid_current_snapshot()) - $xid) > 0")
+	  or die "new snapshot does not have a newer horizon";
+
+	$node_primary->safe_psql('testdb', qq[VACUUM $vac_option verbose $to_vac;
+										  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal;]);
+}
 ########################
 # Initialize primary node
 ########################
@@ -248,6 +270,7 @@ $node_primary->append_conf(
 wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
+autovacuum = off
 });
 $node_primary->dump_info;
 $node_primary->start;
@@ -464,22 +487,17 @@ $node_subscriber->stop;
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM full pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('full', 'CREATE TABLE conflict_test(x integer, y text);
+								 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
@@ -546,22 +564,17 @@ my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
+	0, 1, 1);
 
 # This should trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  DROP TABLE conflict_test;
-  VACUUM pg_class;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 DROP TABLE conflict_test;', 'pg_class');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -584,23 +597,18 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('row_removal_',
+	'shared_row_removal_', 0, 1, 1);
 
 # Trigger the conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE ROLE create_trash;
-  DROP ROLE create_trash;
-  VACUUM pg_authid;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE ROLE create_trash;
+							 DROP ROLE create_trash;', 'pg_authid');
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 1);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -621,18 +629,14 @@ check_pg_recvlogical_stderr($handle,
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+recreate_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
+	'no_conflict_', 0, 1, 1);
 
 # This should not trigger a conflict
-$node_primary->safe_psql(
-	'testdb', qq[
-  CREATE TABLE conflict_test(x integer, y text);
-  INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
-  UPDATE conflict_test set x=1, y=1;
-  VACUUM conflict_test;
-  INSERT INTO flush_wal DEFAULT VALUES; -- see create table flush_wal
-]);
+wait_until_vacuum_can_remove('', 'CREATE TABLE conflict_test(x integer, y text);
+							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
+							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
+
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # message should not be issued
@@ -679,8 +683,8 @@ $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+recreate_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -695,7 +699,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify conflict_reason is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -739,7 +743,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify conflict_reason is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.34.1


--y6m3n3Pgvo2G5ZzY--





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-19 06:42  Amit Kapila <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Amit Kapila @ 2025-03-19 06:42 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: [email protected]

On Mon, Feb 10, 2025 at 8:12 PM Bertrand Drouvot
<[email protected]> wrote:
>
> Please find attached a patch to $SUBJECT.
>
> In rare circumstances (and on slow machines) it is possible that a xl_running_xacts
> is emitted and that the catalog_xmin of a logical slot on the standby advances
> past the conflict point. In that case, no conflict is reported and the test
> fails. It has been observed several times and the last discussion can be found
> in [1].
>

Is my understanding correct that bgwriter on primary node has created
a  xl_running_xacts, then that record is replicated to standby, and
while decoding it (xl_running_xacts) on standby via active_slot, we
advanced the catalog_xmin of active_slot? If this happens then the
replay of vacuum record on standby won't be able to invalidate the
active slot, right?

So, if the above is correct, the reason for generating extra
xl_running_xacts on primary is Vacuum followed by Insert on primary
via below part of test:
$node_primary->safe_psql(
'testdb', qq[VACUUM $vac_option verbose $to_vac;
  INSERT INTO flush_wal DEFAULT VALUES;]);

> To avoid the race condition to occur this commit adds an injection point to prevent
> the catalog_xmin of a logical slot to advance past the conflict point.
>
> While working on this patch, some adjustements have been needed for injection
> points (they are proposed in 0001):
>
> - Adds the ability to wakeup() and detach() while ensuring that no process can
> wait in between. It's done thanks to a new injection_points_wakeup_detach()
> function that is holding the spinlock during the whole duration.
>
> - If the walsender is waiting on the injection point and that the logical slot
> is conflicting, then the walsender process is killed and so it is not able to
> "empty" it's injection slot. So the next injection_wait() should reuse this slot
> (instead of using an empty one). injection_wait() has been modified that way
> in 0001.
>
> With 0001 in place, then we can make use of an injection point in
> LogicalConfirmReceivedLocation() and update 035_standby_logical_decoding.pl to
> prevent the catalog_xmin of a logical slot to advance past the conflict point.
>
> Remarks:
>
> R1. The issue still remains in v16 though (as injection points are available since
> v17).
>

This is not idle case because the test would still keep failing
intermittently on 16. I am wondering what if we start a transaction
before vacuum and do some DML in it but didn't commit that xact till
the active_slot test is finished then even the extra logging of
xl_running_xacts shouldn't advance xmin during decoding. This is
because reorder buffer may point to the xmin before vacuum. See
following code:

SnapBuildProcessRunningXacts()
....
xmin = ReorderBufferGetOldestXmin(builder->reorder);
if (xmin == InvalidTransactionId)
xmin = running->oldestRunningXid;
elog(DEBUG3, "xmin: %u, xmax: %u, oldest running: %u, oldest xmin: %u",
builder->xmin, builder->xmax, running->oldestRunningXid, xmin);
LogicalIncreaseXminForSlot(lsn, xmin);
...

Note that I have not tested this case, so I could be wrong. But if
possible, we should try to find some solution which could be
backpatched to 16 as well.

-- 
With Regards,
Amit Kapila.

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-19 10:26  Bertrand Drouvot <[email protected]>
  parent: Amit Kapila <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Bertrand Drouvot @ 2025-03-19 10:26 UTC (permalink / raw)
  To: Amit Kapila <[email protected]>; +Cc: [email protected]

Hi,

On Wed, Mar 19, 2025 at 12:12:19PM +0530, Amit Kapila wrote:
> On Mon, Feb 10, 2025 at 8:12 PM Bertrand Drouvot
> <[email protected]> wrote:
> >
> > Please find attached a patch to $SUBJECT.
> >
> > In rare circumstances (and on slow machines) it is possible that a xl_running_xacts
> > is emitted and that the catalog_xmin of a logical slot on the standby advances
> > past the conflict point. In that case, no conflict is reported and the test
> > fails. It has been observed several times and the last discussion can be found
> > in [1].
> >
> 

Thanks for looking at it!

> Is my understanding correct that bgwriter on primary node has created
> a  xl_running_xacts, then that record is replicated to standby, and
> while decoding it (xl_running_xacts) on standby via active_slot, we
> advanced the catalog_xmin of active_slot? If this happens then the
> replay of vacuum record on standby won't be able to invalidate the
> active slot, right?

Yes, that's also my understanding. It's also easy to "simulate" by adding
a checkpoint on the primary and a long enough sleep after we launched our sql in 
wait_until_vacuum_can_remove().

> So, if the above is correct, the reason for generating extra
> xl_running_xacts on primary is Vacuum followed by Insert on primary
> via below part of test:
> $node_primary->safe_psql(
> 'testdb', qq[VACUUM $vac_option verbose $to_vac;
>   INSERT INTO flush_wal DEFAULT VALUES;]);

I'm not sure, I think a xl_running_xacts could also be generated (for example by
the checkpointer) before the vacuum (should the system be slow enough).

> > Remarks:
> >
> > R1. The issue still remains in v16 though (as injection points are available since
> > v17).
> >
> 
> This is not idle case because the test would still keep failing
> intermittently on 16.

I do agree.

> I am wondering what if we start a transaction
> before vacuum and do some DML in it but didn't commit that xact till
> the active_slot test is finished then even the extra logging of
> xl_running_xacts shouldn't advance xmin during decoding.

I'm not sure, as I think a xl_running_xacts could still be generated after
we execute "our sql" meaning:

"
    $node_primary->safe_psql('testdb', qq[$sql]);
"

and before we launch the new DML. In that case I guess the issue could still
happen.

OTOH If we create the new DML "before" we launch "our sql" then the test
would also fail for both active and inactive slots because that would not
invalidate any slots.

I did observe the above with the attached changes (just changing the PREPARE
TRANSACTION location).

> we should try to find some solution which could be
> backpatched to 16 as well.

I agree, but I'm not sure it's doable as it looks to me that we should prevent
the catalog xmin to advance to advance past the conflict point while still
generating a conflict point. Will try to give it another thought.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..edd8009fbce 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -258,10 +258,20 @@ sub wait_until_vacuum_can_remove
 	# Launch our sql.
 	$node_primary->safe_psql('testdb', qq[$sql]);
 
+	$node_primary->safe_psql('testdb',"CHECKPOINT");
+	sleep(20);
+
+	$node_primary->safe_psql(
+	    'testdb', "
+		BEGIN;
+		PREPARE TRANSACTION 'prevent_slot_advance_v1';
+		INSERT INTO prevent_slot_advance VALUES (1);"
+	);
+
 	# Wait until we get a newer horizon.
-	$node_primary->poll_query_until('testdb',
-		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"
-	) or die "new snapshot does not have a newer horizon";
+	#$node_primary->poll_query_until('testdb',
+	#	"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"
+	#) or die "new snapshot does not have a newer horizon";
 
 	# Launch the vacuum command and insert into flush_wal (see CREATE
 	# TABLE flush_wal for the reason).
@@ -281,7 +291,9 @@ wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
 autovacuum = off
+max_prepared_transactions = 10
 });
+
 $node_primary->dump_info;
 $node_primary->start;
 
@@ -305,6 +317,7 @@ $node_primary->backup($backup_name);
 # Some tests need to wait for VACUUM to be replayed. But vacuum does not flush
 # WAL. An insert into flush_wal outside transaction does guarantee a flush.
 $node_primary->psql('testdb', q[CREATE TABLE flush_wal();]);
+$node_primary->psql('testdb', q[CREATE TABLE prevent_slot_advance(a int);]);
 
 #######################
 # Initialize standby node
@@ -565,6 +578,8 @@ check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 # Attempting to alter an invalidated slot should result in an error
 ($result, $stdout, $stderr) = $node_standby->psql(
 	'postgres',
@@ -664,6 +679,8 @@ check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 $handle =
   make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
 
@@ -699,6 +716,8 @@ check_for_invalidation('shared_row_removal_', $logstart,
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 $handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
 	\$stderr);
 
@@ -737,6 +756,8 @@ ok( !$node_standby->log_contains(
 	'activeslot slot invalidation is not logged with vacuum on conflict_test'
 );
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 # Verify that pg_stat_database_conflicts.confl_active_logicalslot has not been updated
 ok( $node_standby->poll_query_until(
 		'postgres',


Attachments:

  [text/plain] test_prepared_txn.txt (3.3K, ../../Z9qbvRt1ghOPvS1%[email protected]/2-test_prepared_txn.txt)
  download | inline diff:
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..edd8009fbce 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -258,10 +258,20 @@ sub wait_until_vacuum_can_remove
 	# Launch our sql.
 	$node_primary->safe_psql('testdb', qq[$sql]);
 
+	$node_primary->safe_psql('testdb',"CHECKPOINT");
+	sleep(20);
+
+	$node_primary->safe_psql(
+	    'testdb', "
+		BEGIN;
+		PREPARE TRANSACTION 'prevent_slot_advance_v1';
+		INSERT INTO prevent_slot_advance VALUES (1);"
+	);
+
 	# Wait until we get a newer horizon.
-	$node_primary->poll_query_until('testdb',
-		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"
-	) or die "new snapshot does not have a newer horizon";
+	#$node_primary->poll_query_until('testdb',
+	#	"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"
+	#) or die "new snapshot does not have a newer horizon";
 
 	# Launch the vacuum command and insert into flush_wal (see CREATE
 	# TABLE flush_wal for the reason).
@@ -281,7 +291,9 @@ wal_level = 'logical'
 max_replication_slots = 4
 max_wal_senders = 4
 autovacuum = off
+max_prepared_transactions = 10
 });
+
 $node_primary->dump_info;
 $node_primary->start;
 
@@ -305,6 +317,7 @@ $node_primary->backup($backup_name);
 # Some tests need to wait for VACUUM to be replayed. But vacuum does not flush
 # WAL. An insert into flush_wal outside transaction does guarantee a flush.
 $node_primary->psql('testdb', q[CREATE TABLE flush_wal();]);
+$node_primary->psql('testdb', q[CREATE TABLE prevent_slot_advance(a int);]);
 
 #######################
 # Initialize standby node
@@ -565,6 +578,8 @@ check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 # Attempting to alter an invalidated slot should result in an error
 ($result, $stdout, $stderr) = $node_standby->psql(
 	'postgres',
@@ -664,6 +679,8 @@ check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 $handle =
   make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
 
@@ -699,6 +716,8 @@ check_for_invalidation('shared_row_removal_', $logstart,
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 $handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
 	\$stderr);
 
@@ -737,6 +756,8 @@ ok( !$node_standby->log_contains(
 	'activeslot slot invalidation is not logged with vacuum on conflict_test'
 );
 
+$node_primary->safe_psql('testdb', "COMMIT PREPARED 'prevent_slot_advance_v1'");
+
 # Verify that pg_stat_database_conflicts.confl_active_logicalslot has not been updated
 ok( $node_standby->poll_query_until(
 		'postgres',


^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-21 12:28  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-03-21 12:28 UTC (permalink / raw)
  To: 'Bertrand Drouvot' <[email protected]>; Amit Kapila <[email protected]>; +Cc: [email protected] <[email protected]>

Dear Bertrand,

I'm also working on the thread to resolve the random failure.

> Yes, that's also my understanding. It's also easy to "simulate" by adding
> a checkpoint on the primary and a long enough sleep after we launched our sql in
> wait_until_vacuum_can_remove().

Thanks for letting me know. For me, it could be reporoduced only the sleep().

> > So, if the above is correct, the reason for generating extra
> > xl_running_xacts on primary is Vacuum followed by Insert on primary
> > via below part of test:
> > $node_primary->safe_psql(
> > 'testdb', qq[VACUUM $vac_option verbose $to_vac;
> >   INSERT INTO flush_wal DEFAULT VALUES;]);
> 
> I'm not sure, I think a xl_running_xacts could also be generated (for example by
> the checkpointer) before the vacuum (should the system be slow enough).

I think you are right. When I added `CHECKPOINT` and sleep after the user SQLs,
I got the below ordering. This meant that RUNNING_XACTS are generated before the
prune triggered by the vacuum.
```
...
lsn: 0/04025218, prev 0/040251A0, desc: RUNNING_XACTS nextXid 766 latestCompletedXid 765 oldestRunningXid 766
...
lsn: 0/04028FD0, prev 0/04026FB0, desc: PRUNE_ON_ACCESS snapshotConflictHorizon: 765,...
...
```

> I'm not sure, as I think a xl_running_xacts could still be generated after
> we execute "our sql" meaning:
> 
> "
>     $node_primary->safe_psql('testdb', qq[$sql]);
> "
> 
> and before we launch the new DML. In that case I guess the issue could still
> happen.
> 
> OTOH If we create the new DML "before" we launch "our sql" then the test
> would also fail for both active and inactive slots because that would not
> invalidate any slots.
> 
> I did observe the above with the attached changes (just changing the PREPARE
> TRANSACTION location).

I've also tried the idea with the living transaction via background_psql(),
but I got the same result. The test could fail when RUNNING_XACTS record was
generated before the transaction starts.

> I agree, but I'm not sure it's doable as it looks to me that we should prevent
> the catalog xmin to advance to advance past the conflict point while still
> generating a conflict point. Will try to give it another thought.

One primitive idea for me was to stop the walsender/pg_recvlogical process for a while.
SIGSTOP signal for pg_recvlogical may do the idea, but ISTM it could not be on windows.
See 019_replslot_limit.pl.

Best regards,
Hayato Kuroda
FUJITSU LIMITED 

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-21 16:18  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 2 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-03-21 16:18 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>

Hi Kuroda-san,

On Fri, Mar 21, 2025 at 12:28:10PM +0000, Hayato Kuroda (Fujitsu) wrote:
> I'm also working on the thread to resolve the random failure.

Thanks for looking at it!

> I've also tried the idea with the living transaction via background_psql(),
> but I got the same result. The test could fail when RUNNING_XACTS record was
> generated before the transaction starts.

and thanks for testing and confirming too.

> SIGSTOP signal for pg_recvlogical may do the idea,

Yeah, but would we be "really" testing an "active" slot?

At the end we want to produce an invalidation that may or not happen on a real
environment. The corner case is in the test, not an issue of the feature to
fix.

So, I'm not sure I like the idea that much, but thinking out loud: I wonder if
we could bypass the "active" slot checks in 16 and 17 and use injection points as 
proposed as of 18 (as we need the injection points changes proposed in 0001
up-thread). Thoughts?

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-24 04:54  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-03-24 04:54 UTC (permalink / raw)
  To: 'Bertrand Drouvot' <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>

Dear Bertrand,

> > SIGSTOP signal for pg_recvlogical may do the idea,
> 
> Yeah, but would we be "really" testing an "active" slot?

Yeah, this is also a debatable point.

> At the end we want to produce an invalidation that may or not happen on a real
> environment. The corner case is in the test, not an issue of the feature to
> fix.

I also think this is the test-issue, not the codebase.

> So, I'm not sure I like the idea that much, but thinking out loud: I wonder if
> we could bypass the "active" slot checks in 16 and 17 and use injection points as
> proposed as of 18 (as we need the injection points changes proposed in 0001
> up-thread). Thoughts?

I do not have other idea neither. I checked your patch set could solve the issue.

Comments for the patch:
I'm not sure whether new API is really needed. Isn't it enough to use both
injection_points_wakeup() and injection_points_detach()? This approach does not
require bumping the version, and can be backported to PG17.

Also, another check whether the extension can be installed for the node is required.
Please see 041_checkpoint_at_promote.pl.

Best regards,
Hayato Kuroda
FUJITSU LIMITED

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-25 03:44  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-03-25 03:44 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: Amit Kapila <[email protected]>; [email protected] <[email protected]>

Hi Kuroda-san,

On Mon, Mar 24, 2025 at 04:54:21AM +0000, Hayato Kuroda (Fujitsu) wrote:
> > So, I'm not sure I like the idea that much, but thinking out loud: I wonder if
> > we could bypass the "active" slot checks in 16 and 17 and use injection points as
> > proposed as of 18 (as we need the injection points changes proposed in 0001
> > up-thread). Thoughts?
> 
> I do not have other idea neither. I checked your patch set could solve the issue.

Thanks for looking at it!

> Comments for the patch:
> I'm not sure whether new API is really needed. Isn't it enough to use both
> injection_points_wakeup() and injection_points_detach()?

I think that the proposed changes are needed as they fix 2 issues that I hit
while working on 0002:

1. ensure that no process can wait in between wakeup() and detach().

2. If the walsender is waiting on the injection point and the logical slot
is conflicting, then the walsender process is killed and so it is not able to
"empty" it's injection slot. So the next injection_wait() should reuse this slot
(instead of using an empty one).

> Also, another check whether the extension can be installed for the node is required.
> Please see 041_checkpoint_at_promote.pl.

Indeed I can see the "# Check if the extension injection_points is available, as
it may be possible that this script is run with installcheck, where the module
would not be installed by default", in 041_checkpoint_at_promote.pl.

Thanks! I think that makes sense and will add it in the proposed patch (early
next week).

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-25 11:34  Amit Kapila <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 2 replies; 55+ messages in thread

From: Amit Kapila @ 2025-03-25 11:34 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; [email protected] <[email protected]>

On Fri, Mar 21, 2025 at 9:48 PM Bertrand Drouvot
<[email protected]> wrote:
>
> So, I'm not sure I like the idea that much, but thinking out loud: I wonder if
> we could bypass the "active" slot checks in 16 and 17 and use injection points as
> proposed as of 18 (as we need the injection points changes proposed in 0001
> up-thread). Thoughts?
>

The key point is that snapshotConflictHorizon should always be greater
than or equal to oldestRunningXid for this test to pass. The challenge
is that vacuum LOGs the safest xid to be removed as
snapshotConflictHorizon, which I think will always be either one or
more lesser than oldestRunningXid. So, we can't make it pass unless we
ensure there is no running_xact record gets logged after the last
successful transaction (in this case SQL passed to function
wait_until_vacuum_can_remove) and the till the vacuum is replayed on
the standby. I see even check_for_invalidation('pruning_', $logstart,
'with on-access pruning'); failed  [1].

Seeing all these failures, I wonder whether we can reliably test
active slots apart from wal_level change test (aka Scenario 6:
incorrect wal_level on primary.). Sure, we can try by having some
injection point kind of tests, but is it really worth because, anyway
the active slots won't get invalidated in the scenarios for row
removal we are testing in this case. The other possibility is to add a
developer-level debug_disable_running_xact GUC to test this and
similar cases, or can't we have an injection point to control logging
this WAL record? I have seen the need to control logging running_xact
record in other cases as well.

[1] - https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=skink&dt=2025-03-19%2007%3A08%3A16

-- 
With Regards,
Amit Kapila.

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-26 07:47  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Amit Kapila <[email protected]>
  1 sibling, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-03-26 07:47 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; Bertrand Drouvot <[email protected]>; +Cc: [email protected] <[email protected]>

Dear Amit, Bertrand,

> Seeing all these failures, I wonder whether we can reliably test
> active slots apart from wal_level change test (aka Scenario 6:
> incorrect wal_level on primary.). Sure, we can try by having some
> injection point kind of tests, but is it really worth because, anyway
> the active slots won't get invalidated in the scenarios for row
> removal we are testing in this case. The other possibility is to add a
> developer-level debug_disable_running_xact GUC to test this and
> similar cases, or can't we have an injection point to control logging
> this WAL record? I have seen the need to control logging running_xact
> record in other cases as well.

Based on the idea which controls generating RUNNING_XACTS, I prototyped a patch.
When the instance is attached the new injection point, all processes would skip
logging the record. This does not need to extend injection_point module.
I tested with reproducer and passed on my env.
Sadly IS_INJECTION_POINT_ATTACHED() was introduced for PG18 so that the patch
could not backport for PG17 as-is.

How do you feel?

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] 0001-Use-injection_point-to-stabilize-035_standby_logical.patch (5.9K, ../../OS7PR01MB14968CC605A5FCDD678AD7661F5A62@OS7PR01MB14968.jpnprd01.prod.outlook.com/2-0001-Use-injection_point-to-stabilize-035_standby_logical.patch)
  download | inline diff:
From 1f439e0c6cadc952eecbcded2d2d249d9fec9d36 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 14:19:50 +0900
Subject: [PATCH] Use injection_point to stabilize 035_standby_logical_decoding

---
 src/backend/storage/ipc/standby.c             | 16 ++++++++
 .../t/035_standby_logical_decoding.pl         | 39 +++++++++++++++----
 2 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index 5acb4508f85..35056eee67b 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -31,6 +31,7 @@
 #include "storage/sinvaladt.h"
 #include "storage/standby.h"
 #include "utils/hsearch.h"
+#include "utils/injection_point.h"
 #include "utils/ps_status.h"
 #include "utils/timeout.h"
 #include "utils/timestamp.h"
@@ -1287,6 +1288,21 @@ LogStandbySnapshot(void)
 
 	Assert(XLogStandbyInfoActive());
 
+	/* For testing slot invalidation due to the conflict */
+#ifdef USE_INJECTION_POINTS
+	if (IS_INJECTION_POINT_ATTACHED("log-running-xacts"))
+	{
+		/*
+		 * In 035_standby_logical_decoding.pl, RUNNING_XACTS could move slots's
+		 * xmin forward and cause random failures. Skip generating to avoid it.
+		 *
+		 * XXX What value should we return here? Originally this returns the
+		 * inserted location of RUNNING_XACT record. Based on that, here
+		 * returns the latest insert location for now.
+		 */
+		return GetInsertRecPtr();
+	}
+#endif
 	/*
 	 * Get details of any AccessExclusiveLocks being held at the moment.
 	 */
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..1a721744ef0 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -10,6 +10,11 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
 my ($stdout, $stderr, $cascading_stdout, $cascading_stderr, $handle);
 
 my $node_primary = PostgreSQL::Test::Cluster->new('primary');
@@ -251,6 +256,11 @@ sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
 
+	# Note that from this point the checkpointer and bgwriter will wait before
+	# they write RUNNING_XACT record.
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_attach('log-running-xacts', 'wait');");
+
 	# Get the current xid horizon,
 	my $xid_horizon = $node_primary->safe_psql('testdb',
 		qq[select pg_snapshot_xmin(pg_current_snapshot());]);
@@ -258,6 +268,10 @@ sub wait_until_vacuum_can_remove
 	# Launch our sql.
 	$node_primary->safe_psql('testdb', qq[$sql]);
 
+	# XXX If the instance does not attach 'log-running-xacts', the bgwriter
+	# pocess would generate RUNNING_XACTS record, so that the test would fail.
+	sleep(20);
+
 	# Wait until we get a newer horizon.
 	$node_primary->poll_query_until('testdb',
 		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"
@@ -268,6 +282,12 @@ sub wait_until_vacuum_can_remove
 	$node_primary->safe_psql(
 		'testdb', qq[VACUUM $vac_option verbose $to_vac;
 										  INSERT INTO flush_wal DEFAULT VALUES;]);
+
+	$node_primary->wait_for_replay_catchup($node_standby);
+
+	# Resume working processes
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_detach('log-running-xacts');");
 }
 
 ########################
@@ -285,6 +305,14 @@ autovacuum = off
 $node_primary->dump_info;
 $node_primary->start;
 
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node_primary->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
 $node_primary->psql('postgres', q[CREATE DATABASE testdb]);
 
 $node_primary->safe_psql('testdb',
@@ -528,6 +556,9 @@ is($result, qq(10), 'check replicated inserts after subscription on standby');
 $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
+# Create the injection_points extension
+$node_primary->safe_psql('testdb', 'CREATE EXTENSION injection_points;');
+
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
@@ -557,8 +588,6 @@ wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
 								 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 
@@ -656,8 +685,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
 							 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 
@@ -690,8 +717,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE ROLE create_trash;
 							 DROP ROLE create_trash;', 'pg_authid');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid');
@@ -724,8 +749,6 @@ wait_until_vacuum_can_remove(
 							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
 							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # message should not be issued
 ok( !$node_standby->log_contains(
 		"invalidating obsolete slot \"no_conflict_inactiveslot\"", $logstart),
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-26 10:56  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Amit Kapila <[email protected]>
  1 sibling, 0 replies; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-03-26 10:56 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; Bertrand Drouvot <[email protected]>; +Cc: [email protected] <[email protected]>

Dear Amit, Bertrand,

> Seeing all these failures, I wonder whether we can reliably test
> active slots apart from wal_level change test (aka Scenario 6:
> incorrect wal_level on primary.).

Hmm, agreed. We do not have good solution to stabilize tests, at least for now.
I've created a patch for PG16 which avoids using active slots in scenario 1, 2, 3,
and 5 like attached. Other tests still use active slots:

* Scenario 6 invalidate slots due to the incorrect wal_level, so it retained.
* 'behaves_ok_' testcase, scenario 4 and 'Test standby promotion...' testcase
  won't invalidate slots, so they retained.
* 'DROP DATABASE should drops...' invalidates slots, but it does not related with
  xmin horizon, so it retained.

The patch aimed only PG16, but can be created for PG17 as well, if needed.

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] 0001-Avoid-using-active-slots-in-035_standby_logical_deco.patch (14.4K, ../../OSCPR01MB14966CB132168D1B5554E8AA6F5A62@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-0001-Avoid-using-active-slots-in-035_standby_logical_deco.patch)
  download | inline diff:
From 8e4375d389bfbf68a3e8d19f4d69a0183c994b5b Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH] Avoid using active slots in 035_standby_logical_decoding

---
 .../t/035_standby_logical_decoding.pl         | 164 +++++++++---------
 1 file changed, 82 insertions(+), 82 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8120dfc2132..9ab08d75fd3 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $needs_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -191,22 +201,22 @@ sub check_slots_conflicting_status
 	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +225,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +235,23 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -262,6 +276,10 @@ sub wait_until_vacuum_can_remove
 	# Launch our sql.
 	$node_primary->safe_psql('testdb', qq[$sql]);
 
+	# XXX: Reproducer - must be removed before being pushed
+	$node_primary->safe_psql('testdb', 'CHECKPOINT');
+	sleep(20);
+
 	# Wait until we get a newer horizon.
 	$node_primary->poll_query_until('testdb',
 		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"
@@ -389,7 +407,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -536,11 +554,13 @@ $node_subscriber->stop;
 # Scenario 1: hot_standby_feedback off and vacuum FULL
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -550,19 +570,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -580,7 +592,7 @@ check_slots_conflicting_status(1);
 
 # Get the restart_lsn from an invalidated slot
 my $restart_lsn = $node_standby->safe_psql('postgres',
-	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_activeslot' and conflicting is true;"
+	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_inactiveslot' and conflicting is true;"
 );
 
 chomp($restart_lsn);
@@ -615,14 +627,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -632,32 +646,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -668,29 +676,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -733,13 +735,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -754,17 +758,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -777,10 +777,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -798,7 +798,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -830,10 +830,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -897,14 +897,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-27 10:20  Amit Kapila <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Amit Kapila @ 2025-03-27 10:20 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; [email protected] <[email protected]>

On Wed, Mar 26, 2025 at 1:17 PM Hayato Kuroda (Fujitsu)
<[email protected]> wrote:
>
> > Seeing all these failures, I wonder whether we can reliably test
> > active slots apart from wal_level change test (aka Scenario 6:
> > incorrect wal_level on primary.). Sure, we can try by having some
> > injection point kind of tests, but is it really worth because, anyway
> > the active slots won't get invalidated in the scenarios for row
> > removal we are testing in this case. The other possibility is to add a
> > developer-level debug_disable_running_xact GUC to test this and
> > similar cases, or can't we have an injection point to control logging
> > this WAL record? I have seen the need to control logging running_xact
> > record in other cases as well.
>
> Based on the idea which controls generating RUNNING_XACTS, I prototyped a patch.
> When the instance is attached the new injection point, all processes would skip
> logging the record. This does not need to extend injection_point module.
>

Right, I think this is a better idea. I have a few comments:
1.
+ /*
+ * In 035_standby_logical_decoding.pl, RUNNING_XACTS could move slots's
+ * xmin forward and cause random failures.

No need to use test file name in code comments.

2. The comments atop wait_until_vacuum_can_remove can be changed to
indicate that we will avoid logging running_xact with the help of
injection points.

3.
+ # Note that from this point the checkpointer and bgwriter will wait before
+ # they write RUNNING_XACT record.
+ $node_primary->safe_psql('testdb',
+ "SELECT injection_points_attach('log-running-xacts', 'wait');");

Isn't it better to use 'error' as the second parameter as we don't
want to wait at this injection point?

4.
+ # XXX If the instance does not attach 'log-running-xacts', the bgwriter
+ # pocess would generate RUNNING_XACTS record, so that the test would fail.
+ sleep(20);

I think it is better to make a separate patch (as a first patch) for
this so that it can be used as a reproducer. I suggest to use
checkpoint as used by one of Bertrand's patches to ensure that the
issue reproduces in every environment.

> Sadly IS_INJECTION_POINT_ATTACHED() was introduced for PG18 so that the patch
> could not backport for PG17 as-is.
>

We can use 'wait' mode API in PG17 as used in one of the tests
(injection_points_attach('heap_update-before-pin', 'wait');) but I
think it may be better to just leave testing active slots in
backbranches because anyway the new development happens on HEAD and we
want to ensure that no breakage happens there.

-- 
With Regards,
Amit Kapila.





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-28 09:02  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Amit Kapila <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-03-28 09:02 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; [email protected] <[email protected]>

Dear Amit,

> 
> Right, I think this is a better idea. I have a few comments:
> 1.
> + /*
> + * In 035_standby_logical_decoding.pl, RUNNING_XACTS could move slots's
> + * xmin forward and cause random failures.
> 
> No need to use test file name in code comments.

Fixed.

> 2. The comments atop wait_until_vacuum_can_remove can be changed to
> indicate that we will avoid logging running_xact with the help of
> injection points.

Comments were updated for the master. In back-branches, they were removed
because the risk was removed.

> 3.
> + # Note that from this point the checkpointer and bgwriter will wait before
> + # they write RUNNING_XACT record.
> + $node_primary->safe_psql('testdb',
> + "SELECT injection_points_attach('log-running-xacts', 'wait');");
> 
> Isn't it better to use 'error' as the second parameter as we don't
> want to wait at this injection point?

Right, and the comment atop it was updated.

> 4.
> + # XXX If the instance does not attach 'log-running-xacts', the bgwriter
> + # pocess would generate RUNNING_XACTS record, so that the test would fail.
> + sleep(20);
> 
> I think it is better to make a separate patch (as a first patch) for
> this so that it can be used as a reproducer. I suggest to use
> checkpoint as used by one of Bertrand's patches to ensure that the
> issue reproduces in every environment.

Reproducer was separated to the .txt file.

> > Sadly IS_INJECTION_POINT_ATTACHED() was introduced for PG18 so that the
> patch
> > could not backport for PG17 as-is.
> >
> 
> We can use 'wait' mode API in PG17 as used in one of the tests
> (injection_points_attach('heap_update-before-pin', 'wait');) but I
> think it may be better to just leave testing active slots in
> backbranches because anyway the new development happens on HEAD and we
> want to ensure that no breakage happens there.

OK. I've attached a patch for PG17 as well. Commit messages for them were also
updated.

Best regards,
Hayato Kuroda
FUJITSU LIMITED


diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index d68a8f9b828..71c3ad896d5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -273,6 +273,9 @@ sub wait_until_vacuum_can_remove
 	# Launch our sql.
 	$node_primary->safe_psql('testdb', qq[$sql]);
 
+	$node_primary->safe_psql('testdb', 'CHECKPOINT');
+	sleep(20);
+
 	# Wait until we get a newer horizon.
 	$node_primary->poll_query_until('testdb',
 		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"


Attachments:

  [text/plain] reproducer.txt (637B, ../../OSCPR01MB1496683CB3BD0B4ACD1BADAAFF5A02@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-reproducer.txt)
  download | inline diff:
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index d68a8f9b828..71c3ad896d5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -273,6 +273,9 @@ sub wait_until_vacuum_can_remove
 	# Launch our sql.
 	$node_primary->safe_psql('testdb', qq[$sql]);
 
+	$node_primary->safe_psql('testdb', 'CHECKPOINT');
+	sleep(20);
+
 	# Wait until we get a newer horizon.
 	$node_primary->poll_query_until('testdb',
 		"SELECT (select pg_snapshot_xmin(pg_current_snapshot())::text::int - $xid_horizon) > 0"


  [application/octet-stream] PG17-v2-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (16.8K, ../../OSCPR01MB1496683CB3BD0B4ACD1BADAAFF5A02@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-PG17-v2-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From ac405310591719db0c37356e4487181bed8b3f71 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH vPG17] Stabilize 035_standby_logical_decoding.pl by using the
 injection_points.

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 201 ++++++++----------
 1 file changed, 91 insertions(+), 110 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..d68a8f9b828 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $needs_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -171,42 +181,46 @@ sub change_hot_standby_feedback_and_wait_for_xmins
 # Check reason for conflict in pg_replication_slots.
 sub check_slots_conflict_reason
 {
-	my ($slot_prefix, $reason) = @_;
+	my ($slot_prefix, $reason, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
-	$res = $node_standby->safe_psql(
-		'postgres', qq(
-			 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
-	);
-
-	is($res, "$reason", "$active_slot reason for conflict is $reason");
-
 	$res = $node_standby->safe_psql(
 		'postgres', qq(
 			 select invalidation_reason from pg_replication_slots where slot_name = '$inactive_slot' and conflicting;)
 	);
 
 	is($res, "$reason", "$inactive_slot reason for conflict is $reason");
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$res = $node_standby->safe_psql(
+			'postgres', qq(
+				select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
+		);
+
+		is($res, "$reason", "$active_slot reason for conflict is $reason");
+	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +229,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,31 +239,29 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
 # launch a VACUUM.  $vac_option is the set of options to be passed to the
 # VACUUM command, $sql the sql to launch before triggering the vacuum and
 # $to_vac the relation to vacuum.
-#
-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +400,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -539,21 +550,19 @@ $node_subscriber->stop;
 # active slot is invalidated.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # Ensure that replication slot stats are not empty before triggering the
 # conflict.
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,27 +571,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -602,7 +595,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
 	'postgres',
 	"SELECT restart_lsn FROM pg_replication_slots
-		WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+		WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );
 
 chomp($restart_lsn);
@@ -634,14 +627,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -651,32 +646,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -687,29 +676,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -758,13 +741,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -779,17 +764,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -802,10 +783,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -823,7 +804,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
@@ -855,10 +836,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -922,14 +903,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



  [application/octet-stream] PG16-v2-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (15.0K, ../../OSCPR01MB1496683CB3BD0B4ACD1BADAAFF5A02@OSCPR01MB14966.jpnprd01.prod.outlook.com/4-PG16-v2-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From 1f22028e50c2929c24fe2e60e09ac6d1448e6e71 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH vPG16_2] Stabilize 035_standby_logical_decoding.pl by using
 the injection_points.

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 167 ++++++++----------
 1 file changed, 78 insertions(+), 89 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8120dfc2132..c5899907d37 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $needs_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -191,22 +201,22 @@ sub check_slots_conflicting_status
 	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +225,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,31 +235,29 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
 # launch a VACUUM.  $vac_option is the set of options to be passed to the
 # VACUUM command, $sql the sql to launch before triggering the vacuum and
 # $to_vac the relation to vacuum.
-#
-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +396,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -536,11 +543,13 @@ $node_subscriber->stop;
 # Scenario 1: hot_standby_feedback off and vacuum FULL
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -550,19 +559,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -580,7 +581,7 @@ check_slots_conflicting_status(1);
 
 # Get the restart_lsn from an invalidated slot
 my $restart_lsn = $node_standby->safe_psql('postgres',
-	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_activeslot' and conflicting is true;"
+	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_inactiveslot' and conflicting is true;"
 );
 
 chomp($restart_lsn);
@@ -615,14 +616,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -632,32 +635,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -668,29 +665,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -733,13 +724,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -754,17 +747,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -777,10 +766,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -798,7 +787,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -830,10 +819,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -897,14 +886,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



  [application/octet-stream] v2-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (6.8K, ../../OSCPR01MB1496683CB3BD0B4ACD1BADAAFF5A02@OSCPR01MB14966.jpnprd01.prod.outlook.com/5-v2-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From 89e336611da6f70e2e57d6093b35ed385021e8ad Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 14:19:50 +0900
Subject: [PATCH v2] Stabilize 035_standby_logical_decoding.pl by using the
 injection_points.

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip generating the record when the instance attached to a new
injection point.

This failure can happen since logical decoding is allowed on the standby server.
But the interface of injection_points we used exists only on master, so we do
not backpatch.
---
 src/backend/storage/ipc/standby.c             | 16 +++++++
 .../t/035_standby_logical_decoding.pl         | 45 +++++++++++++------
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index 5acb4508f85..fd175147a70 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -31,6 +31,7 @@
 #include "storage/sinvaladt.h"
 #include "storage/standby.h"
 #include "utils/hsearch.h"
+#include "utils/injection_point.h"
 #include "utils/ps_status.h"
 #include "utils/timeout.h"
 #include "utils/timestamp.h"
@@ -1287,6 +1288,21 @@ LogStandbySnapshot(void)
 
 	Assert(XLogStandbyInfoActive());
 
+	/* For testing slot invalidation due to the conflict */
+#ifdef USE_INJECTION_POINTS
+	if (IS_INJECTION_POINT_ATTACHED("log-running-xacts"))
+	{
+		/*
+		 * RUNNING_XACTS could move slots's xmin forward and cause random
+		 * failures in some tests. Skip generating to avoid it.
+		 *
+		 * XXX What value should we return here? Originally this returns the
+		 * inserted location of RUNNING_XACT record. Based on that, here
+		 * returns the latest insert location for now.
+		 */
+		return GetInsertRecPtr();
+	}
+#endif
 	/*
 	 * Get details of any AccessExclusiveLocks being held at the moment.
 	 */
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..b0312e7c118 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -10,6 +10,11 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
 my ($stdout, $stderr, $cascading_stdout, $cascading_stderr, $handle);
 
 my $node_primary = PostgreSQL::Test::Cluster->new('primary');
@@ -241,16 +246,19 @@ sub check_for_invalidation
 # VACUUM command, $sql the sql to launch before triggering the vacuum and
 # $to_vac the relation to vacuum.
 #
-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# Note that injection_point is used to avoid the seeing a xl_running_xacts
+# that would advance an active replication slot's catalog_xmin. Advancing
+# the active replication slot's catalog_xmin would break some tests that
+# expect the active slot to conflict with the catalog xmin horizon.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
 
+	# Note that from this point the checkpointer and bgwriter will skip writing
+	# xl_running_xacts record.
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_attach('log-running-xacts', 'error');");
+
 	# Get the current xid horizon,
 	my $xid_horizon = $node_primary->safe_psql('testdb',
 		qq[select pg_snapshot_xmin(pg_current_snapshot());]);
@@ -268,6 +276,12 @@ sub wait_until_vacuum_can_remove
 	$node_primary->safe_psql(
 		'testdb', qq[VACUUM $vac_option verbose $to_vac;
 										  INSERT INTO flush_wal DEFAULT VALUES;]);
+
+	$node_primary->wait_for_replay_catchup($node_standby);
+
+	# Resume generating the xl_running_xacts record
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_detach('log-running-xacts');");
 }
 
 ########################
@@ -285,6 +299,14 @@ autovacuum = off
 $node_primary->dump_info;
 $node_primary->start;
 
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node_primary->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
 $node_primary->psql('postgres', q[CREATE DATABASE testdb]);
 
 $node_primary->safe_psql('testdb',
@@ -528,6 +550,9 @@ is($result, qq(10), 'check replicated inserts after subscription on standby');
 $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
+# Create the injection_points extension
+$node_primary->safe_psql('testdb', 'CREATE EXTENSION injection_points;');
+
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
@@ -557,8 +582,6 @@ wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
 								 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 
@@ -656,8 +679,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
 							 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 
@@ -690,8 +711,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE ROLE create_trash;
 							 DROP ROLE create_trash;', 'pg_authid');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid');
@@ -724,8 +743,6 @@ wait_until_vacuum_can_remove(
 							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
 							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # message should not be issued
 ok( !$node_standby->log_contains(
 		"invalidating obsolete slot \"no_conflict_inactiveslot\"", $logstart),
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-03-31 09:23  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Bertrand Drouvot @ 2025-03-31 09:23 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi Kuroda-san and Amit,

On Fri, Mar 28, 2025 at 09:02:29AM +0000, Hayato Kuroda (Fujitsu) wrote:
> Dear Amit,
> 
> > 
> > Right, I think this is a better idea.

I like it too and the bonus point is that this injection point can be used
in more tests (more use cases).

A few comments:

==== About v2-0001-Stabilize

=== 1

s/to avoid the seeing a xl_running_xacts/to avoid seeing a xl_running_xacts/?

=== 2 (Nit)

/* For testing slot invalidation due to the conflict */

Not sure "due to the conflict" is needed.

==== About PG17-v2-0001

=== 3

The commit message still mentions injection point.

=== 4

-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.

I'd be tempted to not remove this comment but reword it a bit instead. Something
like?

# Note that pg_current_snapshot() is used to get the horizon.  It does
# not generate a Transaction/COMMIT WAL record, decreasing the risk of
# seeing a xl_running_xacts that would advance an active replication slot's
# catalog_xmin.  Advancing the active replication slot's catalog_xmin
# would break some tests that expect the active slot to conflict with
# the catalog xmin horizon. We ensure that active replication slots are not
# created for tests that might produce this race condition though. 

=== 5

The invalidation checks for active slots are kept for the wal_level case. Also
the active slots are still created to test that logical decoding on the standby 
behaves correctly, when no conflict is expected and for the promotion.

The above makes sense to me.

=== 6 (Nit)

In drop_logical_slots(), s/needs_active_slot/drop_active_slot/?

=== 7 (Nit)

In check_slots_conflict_reason(), s/needs_active_slot/checks_active_slot/?

==== About PG16-v2-0001

Same as for PG17-v2-0001.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-01 01:22  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 2 replies; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-01 01:22 UTC (permalink / raw)
  To: 'Bertrand Drouvot' <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Dear Bertrand,

> s/to avoid the seeing a xl_running_xacts/to avoid seeing a xl_running_xacts/?

Fixed.

> 
> === 2 (Nit)
> 
> /* For testing slot invalidation due to the conflict */
> 
> Not sure "due to the conflict" is needed.
>

OK, removed.

> ==== About PG17-v2-0001
> 
> === 3
> 
> The commit message still mentions injection point.

Oh, removed.

> === 4
> 
> -# Note that pg_current_snapshot() is used to get the horizon.  It does
> -# not generate a Transaction/COMMIT WAL record, decreasing the risk of
> -# seeing a xl_running_xacts that would advance an active replication slot's
> -# catalog_xmin.  Advancing the active replication slot's catalog_xmin
> -# would break some tests that expect the active slot to conflict with
> -# the catalog xmin horizon.
> 
> I'd be tempted to not remove this comment but reword it a bit instead. Something
> like?
> 
> # Note that pg_current_snapshot() is used to get the horizon.  It does
> # not generate a Transaction/COMMIT WAL record, decreasing the risk of
> # seeing a xl_running_xacts that would advance an active replication slot's
> # catalog_xmin.  Advancing the active replication slot's catalog_xmin
> # would break some tests that expect the active slot to conflict with
> # the catalog xmin horizon. We ensure that active replication slots are not
> # created for tests that might produce this race condition though.

Added.

> === 6 (Nit)
> 
> In drop_logical_slots(), s/needs_active_slot/drop_active_slot/?

Fixed.

> === 7 (Nit)
> 
> In check_slots_conflict_reason(), s/needs_active_slot/checks_active_slot/?

Fixed.

> ==== About PG16-v2-0001
> 
> Same as for PG17-v2-0001.

I followed all needed changes.

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] v3-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (6.8K, ../../OSCPR01MB14966351EDCB504D63A2D41D0F5AC2@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-v3-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From a6ded0068211e0a304eb803763f632665831a419 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 14:19:50 +0900
Subject: [PATCH v3] Stabilize 035_standby_logical_decoding.pl by using the
 injection_points.

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip generating the record when the instance attached to a new
injection point.

This failure can happen since logical decoding is allowed on the standby server.
But the interface of injection_points we used exists only on master, so we do
not backpatch.
---
 src/backend/storage/ipc/standby.c             | 16 +++++++
 .../t/035_standby_logical_decoding.pl         | 45 +++++++++++++------
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index 5acb4508f85..0e621e9996a 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -31,6 +31,7 @@
 #include "storage/sinvaladt.h"
 #include "storage/standby.h"
 #include "utils/hsearch.h"
+#include "utils/injection_point.h"
 #include "utils/ps_status.h"
 #include "utils/timeout.h"
 #include "utils/timestamp.h"
@@ -1287,6 +1288,21 @@ LogStandbySnapshot(void)
 
 	Assert(XLogStandbyInfoActive());
 
+	/* For testing slot invalidation */
+#ifdef USE_INJECTION_POINTS
+	if (IS_INJECTION_POINT_ATTACHED("log-running-xacts"))
+	{
+		/*
+		 * RUNNING_XACTS could move slots's xmin forward and cause random
+		 * failures in some tests. Skip generating to avoid it.
+		 *
+		 * XXX What value should we return here? Originally this returns the
+		 * inserted location of RUNNING_XACT record. Based on that, here
+		 * returns the latest insert location for now.
+		 */
+		return GetInsertRecPtr();
+	}
+#endif
 	/*
 	 * Get details of any AccessExclusiveLocks being held at the moment.
 	 */
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..93b11a4de1e 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -10,6 +10,11 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
 my ($stdout, $stderr, $cascading_stdout, $cascading_stderr, $handle);
 
 my $node_primary = PostgreSQL::Test::Cluster->new('primary');
@@ -241,16 +246,19 @@ sub check_for_invalidation
 # VACUUM command, $sql the sql to launch before triggering the vacuum and
 # $to_vac the relation to vacuum.
 #
-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# Note that injection_point is used to avoid the seeing the xl_running_xacts
+# that would advance an active replication slot's catalog_xmin. Advancing
+# the active replication slot's catalog_xmin would break some tests that
+# expect the active slot to conflict with the catalog xmin horizon.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
 
+	# Note that from this point the checkpointer and bgwriter will skip writing
+	# xl_running_xacts record.
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_attach('log-running-xacts', 'error');");
+
 	# Get the current xid horizon,
 	my $xid_horizon = $node_primary->safe_psql('testdb',
 		qq[select pg_snapshot_xmin(pg_current_snapshot());]);
@@ -268,6 +276,12 @@ sub wait_until_vacuum_can_remove
 	$node_primary->safe_psql(
 		'testdb', qq[VACUUM $vac_option verbose $to_vac;
 										  INSERT INTO flush_wal DEFAULT VALUES;]);
+
+	$node_primary->wait_for_replay_catchup($node_standby);
+
+	# Resume generating the xl_running_xacts record
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_detach('log-running-xacts');");
 }
 
 ########################
@@ -285,6 +299,14 @@ autovacuum = off
 $node_primary->dump_info;
 $node_primary->start;
 
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node_primary->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
 $node_primary->psql('postgres', q[CREATE DATABASE testdb]);
 
 $node_primary->safe_psql('testdb',
@@ -528,6 +550,9 @@ is($result, qq(10), 'check replicated inserts after subscription on standby');
 $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
+# Create the injection_points extension
+$node_primary->safe_psql('testdb', 'CREATE EXTENSION injection_points;');
+
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
@@ -557,8 +582,6 @@ wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
 								 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 
@@ -656,8 +679,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
 							 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 
@@ -690,8 +711,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE ROLE create_trash;
 							 DROP ROLE create_trash;', 'pg_authid');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid');
@@ -724,8 +743,6 @@ wait_until_vacuum_can_remove(
 							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
 							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # message should not be issued
 ok( !$node_standby->log_contains(
 		"invalidating obsolete slot \"no_conflict_inactiveslot\"", $logstart),
-- 
2.43.5



  [application/octet-stream] PG16-v3-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (15.0K, ../../OSCPR01MB14966351EDCB504D63A2D41D0F5AC2@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-PG16-v3-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From e42d953093251eb008a1322918e5f20d4bd41f2d Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH vPG16-3] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 165 +++++++++---------
 1 file changed, 81 insertions(+), 84 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8120dfc2132..54e9aa76621 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $drop_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($drop_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -191,22 +201,22 @@ sub check_slots_conflicting_status
 	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +225,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +235,23 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -247,10 +261,11 @@ sub check_for_invalidation
 #
 # Note that pg_current_snapshot() is used to get the horizon.  It does
 # not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
+# seeing the xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that active replication slots are not
+# created for tests that might produce this race condition though. 
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +404,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -536,11 +551,13 @@ $node_subscriber->stop;
 # Scenario 1: hot_standby_feedback off and vacuum FULL
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -550,19 +567,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -580,7 +589,7 @@ check_slots_conflicting_status(1);
 
 # Get the restart_lsn from an invalidated slot
 my $restart_lsn = $node_standby->safe_psql('postgres',
-	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_activeslot' and conflicting is true;"
+	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_inactiveslot' and conflicting is true;"
 );
 
 chomp($restart_lsn);
@@ -615,14 +624,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -632,32 +643,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -668,29 +673,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -733,13 +732,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -754,17 +755,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -777,10 +774,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -798,7 +795,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -830,10 +827,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -897,14 +894,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



  [application/octet-stream] PG17-v3-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (16.9K, ../../OSCPR01MB14966351EDCB504D63A2D41D0F5AC2@OSCPR01MB14966.jpnprd01.prod.outlook.com/4-PG17-v3-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From 5e585fbf44f95ed684f9aacb0b8c2ba46267d012 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH vPG17-3] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 199 +++++++++---------
 1 file changed, 94 insertions(+), 105 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..af29e075eaa 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $drop_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($drop_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -171,42 +181,46 @@ sub change_hot_standby_feedback_and_wait_for_xmins
 # Check reason for conflict in pg_replication_slots.
 sub check_slots_conflict_reason
 {
-	my ($slot_prefix, $reason) = @_;
+	my ($slot_prefix, $reason, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
-	$res = $node_standby->safe_psql(
-		'postgres', qq(
-			 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
-	);
-
-	is($res, "$reason", "$active_slot reason for conflict is $reason");
-
 	$res = $node_standby->safe_psql(
 		'postgres', qq(
 			 select invalidation_reason from pg_replication_slots where slot_name = '$inactive_slot' and conflicting;)
 	);
 
 	is($res, "$reason", "$inactive_slot reason for conflict is $reason");
+
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$res = $node_standby->safe_psql(
+			'postgres', qq(
+				select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
+		);
+
+		is($res, "$reason", "$active_slot reason for conflict is $reason");
+	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +229,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +239,23 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -247,10 +265,11 @@ sub check_for_invalidation
 #
 # Note that pg_current_snapshot() is used to get the horizon.  It does
 # not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
+# seeing the xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that active replication slots are not
+# created for tests that might produce this race condition though. 
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +408,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -539,21 +558,19 @@ $node_subscriber->stop;
 # active slot is invalidated.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # Ensure that replication slot stats are not empty before triggering the
 # conflict.
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,27 +579,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -602,7 +603,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
 	'postgres',
 	"SELECT restart_lsn FROM pg_replication_slots
-		WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+		WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );
 
 chomp($restart_lsn);
@@ -634,14 +635,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -651,32 +654,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -687,29 +684,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -758,13 +749,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -779,17 +772,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -802,10 +791,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -823,7 +812,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
@@ -855,10 +844,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -922,14 +911,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-01 08:32  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  1 sibling, 2 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-01 08:32 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi Kuroda-san,

On Tue, Apr 01, 2025 at 01:22:49AM +0000, Hayato Kuroda (Fujitsu) wrote:
> Dear Bertrand,
> 

Thanks for the updated patch!

> > s/to avoid the seeing a xl_running_xacts/to avoid seeing a xl_running_xacts/?
> 
> Fixed.

hmm, not sure as I still can see:

+# Note that injection_point is used to avoid the seeing the xl_running_xacts

=== 1

+                * XXX What value should we return here? Originally this returns the
+                * inserted location of RUNNING_XACT record. Based on that, here
+                * returns the latest insert location for now.
+                */
+               return GetInsertRecPtr();

Looking at the LogStandbySnapshot() that are using the output lsn, i.e:

pg_log_standby_snapshot()
BackgroundWriterMain()
ReplicationSlotReserveWal()

It looks ok to me to use GetInsertRecPtr().

But if we "really" want to produce a "new" WAL record, what about using
LogLogicalMessage()? It could also be used for debugging purpose. Bonus point:
it does not need wal_level to be set to logical. Thoughts?

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-01 11:52  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 0 replies; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-01 11:52 UTC (permalink / raw)
  To: 'Bertrand Drouvot' <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Dear Bertrand,

> > > s/to avoid the seeing a xl_running_xacts/to avoid seeing a xl_running_xacts/?
> >
> > Fixed.

Sorry, I misunderstood your comment and wrongly fixed. I will address in next version.

> === 1
> 
> +                * XXX What value should we return here? Originally this
> returns the
> +                * inserted location of RUNNING_XACT record. Based on that,
> here
> +                * returns the latest insert location for now.
> +                */
> +               return GetInsertRecPtr();
> 
> Looking at the LogStandbySnapshot() that are using the output lsn, i.e:
> 
> pg_log_standby_snapshot()
> BackgroundWriterMain()
> ReplicationSlotReserveWal()
> 
> It looks ok to me to use GetInsertRecPtr().
> 
> But if we "really" want to produce a "new" WAL record, what about using
> LogLogicalMessage()? It could also be used for debugging purpose. Bonus point:
> it does not need wal_level to be set to logical. Thoughts?

Right. Similarly, an SQL function pg_logical_emit_message() is sometimes used for
the testing purpose, advance_wal() and emit_wal( in Cluster.pm. Even so, we have
not found the use-case yet, thus I want to retain now and will update based on
the future needs.

I'll investigate another point [1] and then will post new version.

[1]: https://www.postgresql.org/message-id/CAA4eK1%2Bx5-eOn5%2BMW6FiUjB_1bBCH8jCCARC1uMrx6erZ3J73w%40mail...

Best regards,
Hayato Kuroda
FUJITSU LIMITED 






^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-01 15:15  Bertrand Drouvot <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-01 15:15 UTC (permalink / raw)
  To: Amit Kapila <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; [email protected] <[email protected]>

Hi,

On Tue, Apr 01, 2025 at 04:55:06PM +0530, Amit Kapila wrote:
> On Tue, Apr 1, 2025 at 2:02 PM Bertrand Drouvot
> <[email protected]> wrote:
> 
> > But if we "really" want to produce a "new" WAL record, what about using
> > LogLogicalMessage()?
> >
> 
> We are using injection points for testing purposes, which means the
> caller is aware of skipping the running_xacts record during the test
> run. So, there doesn't seem to be any reason to do anything extra.

Agree, the idea was to provide extra debugging info for the tests. We can just
keep it in mind should we have a need for.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-02 07:16  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  1 sibling, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-02 07:16 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; [email protected] <[email protected]>

Dear Amit, Bertrand,

> You have not added any injection point for the above case. Isn't it
> possible that if running_xact record is logged concurrently to the
> pruning record, it should move the active slot on standby, and the
> same failure should occur in this case as well?

I considered that the timing failure can happen. Reproducer:

```
 $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'D';]);
+$node_primary->safe_psql('testdb', 'CHECKPOINT');
+sleep(20);
 $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
```

And here is my theory...

Firstly, a new table was created with smaller fill factor. Then, after doing UPDATE
three times, the page became full. At fourth UPDATE command (let's say txn4),
the page pruning was done by the backend process and PRUNE_ON_ACCESS was generated.
It requested standbys to discard tuples before the third UPDATE (say txn3),
thus the slot could be invalidated.
However, if a RUNNING_XACTS record is generated between txn3 and txn4, the
oldestRunningXact would be same xid as txn4, and the catalog_xmin of the standby
slot would be advanced till that. Upcoming PRUNE_ON_ACCESS points the txn3 so that
slot invalidation won't happen in this case.

Based on the fact, I've updated to use injection_points for scenario 5. Of course,
PG16/17 patches won't use the active slot for that scenario.

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] PG16-v4-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (15.0K, ../../OSCPR01MB14966755BC3C534A0058EA07FF5AF2@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-PG16-v4-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From 564bea902f3ae85dca414b2537f35d4eca2c2d7d Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v4-PG16] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 165 +++++++++---------
 1 file changed, 81 insertions(+), 84 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8120dfc2132..ca967fb625f 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $drop_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($drop_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -191,22 +201,22 @@ sub check_slots_conflicting_status
 	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +225,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +235,23 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -247,10 +261,11 @@ sub check_for_invalidation
 #
 # Note that pg_current_snapshot() is used to get the horizon.  It does
 # not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
+# seeing the xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that active replication slots are not
+# created for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +404,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -536,11 +551,13 @@ $node_subscriber->stop;
 # Scenario 1: hot_standby_feedback off and vacuum FULL
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -550,19 +567,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -580,7 +589,7 @@ check_slots_conflicting_status(1);
 
 # Get the restart_lsn from an invalidated slot
 my $restart_lsn = $node_standby->safe_psql('postgres',
-	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_activeslot' and conflicting is true;"
+	"SELECT restart_lsn from pg_replication_slots WHERE slot_name = 'vacuum_full_inactiveslot' and conflicting is true;"
 );
 
 chomp($restart_lsn);
@@ -615,14 +624,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -632,32 +643,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -668,29 +673,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -733,13 +732,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -754,17 +755,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -777,10 +774,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -798,7 +795,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -830,10 +827,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -897,14 +894,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



  [application/octet-stream] PG17-v4-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (16.9K, ../../OSCPR01MB14966755BC3C534A0058EA07FF5AF2@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-PG17-v4-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From 5b2b4218433dd52ff4dc19b03e54b37b036bccb2 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v4-PG17] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 199 +++++++++---------
 1 file changed, 94 insertions(+), 105 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..da5ad1b78f2 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,37 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
-	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
+	my ($slot_prefix, $drop_active_slot) = @_;
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	if ($drop_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+			qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -171,42 +181,46 @@ sub change_hot_standby_feedback_and_wait_for_xmins
 # Check reason for conflict in pg_replication_slots.
 sub check_slots_conflict_reason
 {
-	my ($slot_prefix, $reason) = @_;
+	my ($slot_prefix, $reason, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
-	$res = $node_standby->safe_psql(
-		'postgres', qq(
-			 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
-	);
-
-	is($res, "$reason", "$active_slot reason for conflict is $reason");
-
 	$res = $node_standby->safe_psql(
 		'postgres', qq(
 			 select invalidation_reason from pg_replication_slots where slot_name = '$inactive_slot' and conflicting;)
 	);
 
 	is($res, "$reason", "$inactive_slot reason for conflict is $reason");
+
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$res = $node_standby->safe_psql(
+			'postgres', qq(
+				select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
+		);
+
+		is($res, "$reason", "$active_slot reason for conflict is $reason");
+	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
-
-	# drop the logical slots
-	drop_logical_slots($previous_slot_prefix);
+	my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -215,9 +229,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +239,23 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -247,10 +265,11 @@ sub check_for_invalidation
 #
 # Note that pg_current_snapshot() is used to get the horizon.  It does
 # not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
+# seeing the xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that active replication slots are not
+# created for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +408,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -539,21 +558,19 @@ $node_subscriber->stop;
 # active slot is invalidated.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('behaves_ok_', 1);
+
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 0, 1, 0);
 
 # Ensure that replication slot stats are not empty before triggering the
 # conflict.
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,27 +579,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -602,7 +603,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
 	'postgres',
 	"SELECT restart_lsn FROM pg_replication_slots
-		WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+		WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );
 
 chomp($restart_lsn);
@@ -634,14 +635,16 @@ ok(!-f "$standby_walfile",
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('vacuum_full_', 0);
+
 # get the position to search from in the standby logfile
 my $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to create/drop a relation and
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
-reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('row_removal_', 0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -651,32 +654,26 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict on a shared catalog table is to
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
-reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -687,29 +684,23 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('shared_row_removal_', 0);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
-reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
-	'no_conflict_', 0, 1);
+reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 0, 1);
 
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
@@ -758,13 +749,15 @@ $node_standby->restart;
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
+# drop the logical slots used by previous tests
+drop_logical_slots('no_conflict_', 1);
+
 # get the position to search from in the standby logfile
 $logstart = -s $node_standby->logfile;
 
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
-reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+reactive_slots_change_hfs_and_wait_for_xmins('pruning_', 0, 0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -779,17 +772,13 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -802,10 +791,10 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $logstart = -s $node_standby->logfile;
 
 # drop the logical slots
-drop_logical_slots('pruning_');
+drop_logical_slots('pruning_', 0);
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -823,7 +812,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
@@ -855,10 +844,10 @@ check_pg_recvlogical_stderr($handle,
 ##################################################
 
 # drop the logical slots
-drop_logical_slots('wal_level_');
+drop_logical_slots('wal_level_', 1);
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -922,14 +911,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



  [application/octet-stream] v4-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (8.0K, ../../OSCPR01MB14966755BC3C534A0058EA07FF5AF2@OSCPR01MB14966.jpnprd01.prod.outlook.com/4-v4-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From f67f9fea45c6a3319aa05c8c1feac133178fd9e6 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 14:19:50 +0900
Subject: [PATCH v4] Stabilize 035_standby_logical_decoding.pl by using the
 injection_points.

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip generating the record when the instance attached to a new
injection point.

This failure can happen since logical decoding is allowed on the standby server.
But the interface of injection_points we used exists only on master, so we do
not backpatch.
---
 src/backend/storage/ipc/standby.c             | 16 ++++++
 .../t/035_standby_logical_decoding.pl         | 57 ++++++++++++++-----
 2 files changed, 59 insertions(+), 14 deletions(-)

diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index 5acb4508f85..0e621e9996a 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -31,6 +31,7 @@
 #include "storage/sinvaladt.h"
 #include "storage/standby.h"
 #include "utils/hsearch.h"
+#include "utils/injection_point.h"
 #include "utils/ps_status.h"
 #include "utils/timeout.h"
 #include "utils/timestamp.h"
@@ -1287,6 +1288,21 @@ LogStandbySnapshot(void)
 
 	Assert(XLogStandbyInfoActive());
 
+	/* For testing slot invalidation */
+#ifdef USE_INJECTION_POINTS
+	if (IS_INJECTION_POINT_ATTACHED("log-running-xacts"))
+	{
+		/*
+		 * RUNNING_XACTS could move slots's xmin forward and cause random
+		 * failures in some tests. Skip generating to avoid it.
+		 *
+		 * XXX What value should we return here? Originally this returns the
+		 * inserted location of RUNNING_XACT record. Based on that, here
+		 * returns the latest insert location for now.
+		 */
+		return GetInsertRecPtr();
+	}
+#endif
 	/*
 	 * Get details of any AccessExclusiveLocks being held at the moment.
 	 */
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..96dd0340e8d 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -10,6 +10,11 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
 my ($stdout, $stderr, $cascading_stdout, $cascading_stderr, $handle);
 
 my $node_primary = PostgreSQL::Test::Cluster->new('primary');
@@ -241,16 +246,19 @@ sub check_for_invalidation
 # VACUUM command, $sql the sql to launch before triggering the vacuum and
 # $to_vac the relation to vacuum.
 #
-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# Note that injection_point is used to avoid seeing a xl_running_xacts that
+# would advance an active replication slot's catalog_xmin. Advancing the active
+# replication slot's catalog_xmin would break some tests that expect the active
+# slot to conflict with the catalog xmin horizon.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
 
+	# Note that from this point the checkpointer and bgwriter will skip writing
+	# xl_running_xacts record.
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_attach('log-running-xacts', 'error');");
+
 	# Get the current xid horizon,
 	my $xid_horizon = $node_primary->safe_psql('testdb',
 		qq[select pg_snapshot_xmin(pg_current_snapshot());]);
@@ -268,6 +276,12 @@ sub wait_until_vacuum_can_remove
 	$node_primary->safe_psql(
 		'testdb', qq[VACUUM $vac_option verbose $to_vac;
 										  INSERT INTO flush_wal DEFAULT VALUES;]);
+
+	$node_primary->wait_for_replay_catchup($node_standby);
+
+	# Resume generating the xl_running_xacts record
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_detach('log-running-xacts');");
 }
 
 ########################
@@ -285,6 +299,14 @@ autovacuum = off
 $node_primary->dump_info;
 $node_primary->start;
 
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node_primary->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
 $node_primary->psql('postgres', q[CREATE DATABASE testdb]);
 
 $node_primary->safe_psql('testdb',
@@ -528,6 +550,9 @@ is($result, qq(10), 'check replicated inserts after subscription on standby');
 $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
+# Create the injection_points extension
+$node_primary->safe_psql('testdb', 'CREATE EXTENSION injection_points;');
+
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
@@ -557,8 +582,6 @@ wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
 								 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 
@@ -656,8 +679,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
 							 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 
@@ -690,8 +711,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE ROLE create_trash;
 							 DROP ROLE create_trash;', 'pg_authid');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid');
@@ -724,8 +743,6 @@ wait_until_vacuum_can_remove(
 							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
 							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # message should not be issued
 ok( !$node_standby->log_contains(
 		"invalidating obsolete slot \"no_conflict_inactiveslot\"", $logstart),
@@ -773,6 +790,14 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
 	0);
 
+# Injection_point is used to avoid seeing an xl_running_xacts even here. In
+# scenario 5, we verify the case that the backend process detects the page has
+# enough tuples; thus, page pruning happens. If the record is generated just
+# before doing on-pruning, the catalog_xmin of the active slot would be
+# updated; hence, the conflict would not occur.
+$node_primary->safe_psql('testdb',
+	"SELECT injection_points_attach('log-running-xacts', 'error');");
+
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE prun(id integer, s char(2000)) WITH (fillfactor = 75, user_catalog_table = true);]
@@ -785,6 +810,10 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
+# Resume generating the xl_running_xacts record
+$node_primary->safe_psql('testdb',
+	"SELECT injection_points_detach('log-running-xacts');");
+
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
 
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-02 08:36  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 2 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-02 08:36 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi Kuroda-san,

On Wed, Apr 02, 2025 at 07:16:25AM +0000, Hayato Kuroda (Fujitsu) wrote:
> Dear Amit, Bertrand,
> 
> > You have not added any injection point for the above case. Isn't it
> > possible that if running_xact record is logged concurrently to the
> > pruning record, it should move the active slot on standby, and the
> > same failure should occur in this case as well?
> 
> I considered that the timing failure can happen. Reproducer:
> 
> ```
>  $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'D';]);
> +$node_primary->safe_psql('testdb', 'CHECKPOINT');
> +sleep(20);
>  $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
> ```

Yeah, I was going to provide the exact same reproducer and then saw your email.

> Based on the fact, I've updated to use injection_points for scenario 5. Of course,
> PG16/17 patches won't use the active slot for that scenario.

Thanks for the updated patch!

As far v4-0001:

=== 1

+# would advance an active replication slot's catalog_xmin

s/would/could/? I mean the system also needs to be "slow" enough (so the
sleep() in the reproducer)

=== 2

+# Injection_point is used to avoid seeing an xl_running_xacts even here. In
+# scenario 5, we verify the case that the backend process detects the page has
+# enough tuples; thus, page pruning happens. If the record is generated just
+# before doing on-pruning, the catalog_xmin of the active slot would be
+# updated; hence, the conflict would not occur.

Not sure we need to explain what scenario 5 does here, but that does not hurt
if you feel the need.

Also maybe mention the last update in the comment and add some nuance (like
proposed in === 1), something like?

"
# Injection_point is used to avoid seeing a xl_running_xacts here. Indeed,
# if it is generated between the last 2 updates then the catalog_xmin of the active
# slot could be updated; hence, the conflict could not occur.
"

Apart from that the tests looks good to me and all the problematic scenarios
covered.

As far PG17-v4-0001:

=== 3

-# seeing a xl_running_xacts that would advance an active replication slot's
+# seeing the xl_running_xacts that would advance an active replication slot's

why?

=== 4

It looks like that check_slots_conflict_reason() is not called with checks_active_slot
as argument.

=== 5

I think that we could remove the need for the drop_active_slot parameter in
drop_logical_slots() and just check if an active slot exists (and if so drop
it). That said I'm not sure it's worth to go that far for backpatching.

As far PG16-v4:

=== 6

Same as === 3 and === 5 (=== 4 does not apply as check_slots_conflict_reason()
does not exist).

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-02 11:14  Bertrand Drouvot <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-02 11:14 UTC (permalink / raw)
  To: Amit Kapila <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; [email protected] <[email protected]>

Hi,

On Wed, Apr 02, 2025 at 03:04:07PM +0530, Amit Kapila wrote:
> I have changed it based on your suggestions and made a few other
> changes in the comments. Please see attached.

Thanks!

> *
> +  if (IS_INJECTION_POINT_ATTACHED("log-running-xacts"))
> 
> It is better to name the injection point as skip-log-running-xacts as
> that will be appropriate based on its usage.

Agree.

+# Note that the injection_point avoids seeing a xl_running_xacts that could
and
+# Injection_point avoids seeing an xl_running_xacts even here. This is required

s/an xl_running_xacts/a xl_running_xacts/? in the second one? Also I'm not sure
"even here" is needed.

Apart from the above that LGTM.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-02 12:13  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-02 12:13 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; Bertrand Drouvot <[email protected]>; +Cc: [email protected] <[email protected]>

Dear Amit, Bertrand,

> The other idea to simplify the changes for backbranches:
> sub reactive_slots_change_hfs_and_wait_for_xmins
> {
> ...
> +  my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
> 
>   # create the logical slots
> -  create_logical_slots($node_standby, $slot_prefix);
> +  create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
> 
> ...
> +  if ($needs_active_slot)
> +  {
> +    $handle =
> +      make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
> +  }
> 
> What if this function doesn't take input parameter needs_active_slot
> and rather removes the call to make_slot_active? We will call
> make_slot_active only at the required places. This should make the
> changes much less because after that, we don't need to make changes
> related to drop and create. Sure, in some cases, we will test two
> inactive slots instead of one, but I guess that would be the price to
> keep the tests simple and more like HEAD.

Actually, I could not decide which one is better, so let me share both drafts.
V5-PG17-1 uses the previous approach, and v5-PG17-2 uses new proposed one.
Bertrand, which one do you like?

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] v5-PG17-1-0001-Stabilize-035_standby_logical_decoding.pl.patch (13.5K, ../../OSCPR01MB1496631866E979BD510BC5448F5AF2@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-v5-PG17-1-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From 91c6bb335e4aa09e7e631f1cd8786459d6675b93 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v5-PG17-1] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 171 +++++++++---------
 1 file changed, 86 insertions(+), 85 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..1f3ae86c556 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -44,27 +44,50 @@ sub wait_for_xmins
 # Create the required logical slots on standby.
 sub create_logical_slots
 {
-	my ($node, $slot_prefix) = @_;
+	my ($node, $slot_prefix, $needs_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 	$node->create_logical_slot_on_standby($node_primary, qq($inactive_slot),
 		'testdb');
-	$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
-		'testdb');
+
+	if ($needs_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node->create_logical_slot_on_standby($node_primary, qq($active_slot),
+			'testdb');
+	}
+}
+
+# Checks the existence of the active slot. Returns the name if found, otherwise
+# undef.
+sub active_slot_exists
+{
+	my ($slot_prefix) = @_;
+
+	my $active_slot = $slot_prefix . 'activeslot';
+	my $active_slot_info = $node_standby->slot($active_slot);
+
+	return $active_slot_info->{'plugin'} eq '' ? undef : $active_slot;
 }
 
 # Drop the logical slots on standby.
 sub drop_logical_slots
 {
 	my ($slot_prefix) = @_;
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	$node_standby->psql('postgres',
 		qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
-	$node_standby->psql('postgres',
-		qq[SELECT pg_drop_replication_slot('$active_slot')]);
+
+	# Drops the active slot as well, if exists
+	if (active_slot_exists($slot_prefix))
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$node_standby->psql('postgres',
+				qq[SELECT pg_drop_replication_slot('$active_slot')]);
+	}
 }
 
 # Acquire one of the standby logical slots created by create_logical_slots().
@@ -173,40 +196,47 @@ sub check_slots_conflict_reason
 {
 	my ($slot_prefix, $reason) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
-	$res = $node_standby->safe_psql(
-		'postgres', qq(
-			 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
-	);
-
-	is($res, "$reason", "$active_slot reason for conflict is $reason");
-
 	$res = $node_standby->safe_psql(
 		'postgres', qq(
 			 select invalidation_reason from pg_replication_slots where slot_name = '$inactive_slot' and conflicting;)
 	);
 
 	is($res, "$reason", "$inactive_slot reason for conflict is $reason");
+
+	if (active_slot_exists($slot_prefix))
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$res = $node_standby->safe_psql(
+			'postgres', qq(
+				select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
+		);
+
+		is($res, "$reason", "$active_slot reason for conflict is $reason");
+	}
 }
 
-# Drop the slots, re-create them, change hot_standby_feedback,
-# check xmin and catalog_xmin values, make slot active and reset stat.
+# Create slots, change hot_standby_feedback, check xmin and catalog_xmin
+# values, make slot active and reset stat.
 sub reactive_slots_change_hfs_and_wait_for_xmins
 {
-	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated) = @_;
+	my ($previous_slot_prefix, $slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
 
 	# drop the logical slots
 	drop_logical_slots($previous_slot_prefix);
 
 	# create the logical slots
-	create_logical_slots($node_standby, $slot_prefix);
+	create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	if ($needs_active_slot)
+	{
+		$handle =
+		  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+	}
 
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
@@ -217,7 +247,6 @@ sub check_for_invalidation
 {
 	my ($slot_prefix, $log_start, $test_name) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +255,23 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if (active_slot_exists($slot_prefix))
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +284,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that active replication slots are not
+# created for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -389,7 +424,7 @@ $node_standby->safe_psql('postgres',
 ##################################################
 
 # create the logical slots
-create_logical_slots($node_standby, 'behaves_ok_');
+create_logical_slots($node_standby, 'behaves_ok_', 1);
 
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE decoding_test(x integer, y text);]);
@@ -543,17 +578,13 @@ $node_subscriber->stop;
 # launch a vacuum full on pg_class with hot_standby_feedback turned off on
 # the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
-	0, 1);
+	0, 1, 0);
 
 # Ensure that replication slot stats are not empty before triggering the
 # conflict.
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -567,22 +598,6 @@ check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
-$handle =
-  make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -602,7 +617,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
 	'postgres',
 	"SELECT restart_lsn FROM pg_replication_slots
-		WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+		WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );
 
 chomp($restart_lsn);
@@ -641,7 +656,7 @@ my $logstart = -s $node_standby->logfile;
 # launch a vacuum on pg_class with hot_standby_feedback turned off on the
 # standby.
 reactive_slots_change_hfs_and_wait_for_xmins('vacuum_full_', 'row_removal_',
-	0, 1);
+	0, 1, 0);
 
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
@@ -656,14 +671,6 @@ check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
 
-$handle =
-  make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
@@ -676,7 +683,7 @@ $logstart = -s $node_standby->logfile;
 # create/drop a role and launch a vacuum on pg_authid with
 # hot_standby_feedback turned off on the standby.
 reactive_slots_change_hfs_and_wait_for_xmins('row_removal_',
-	'shared_row_removal_', 0, 1);
+	'shared_row_removal_', 0, 1, 0);
 
 # Trigger the conflict
 wait_until_vacuum_can_remove(
@@ -692,14 +699,6 @@ check_for_invalidation('shared_row_removal_', $logstart,
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
 
-$handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
-	\$stderr);
-
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
@@ -747,6 +746,12 @@ is( $node_standby->safe_psql(
 	'f',
 	'Logical slots are reported as non conflicting');
 
+my $tmp = $node_standby->safe_psql(
+		'postgres',
+		q[select slot_name, conflicting from pg_replication_slots
+			where slot_type = 'logical']);
+print $tmp;
+
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 0);
 
@@ -764,7 +769,7 @@ $logstart = -s $node_standby->logfile;
 # One way to produce recovery conflict is to trigger an on-access pruning
 # on a relation marked as user_catalog_table.
 reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
-	0);
+	0, 0);
 
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
@@ -786,10 +791,6 @@ check_slots_conflict_reason('pruning_', 'rows_removed');
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -805,7 +806,7 @@ $logstart = -s $node_standby->logfile;
 drop_logical_slots('pruning_');
 
 # create the logical slots
-create_logical_slots($node_standby, 'wal_level_');
+create_logical_slots($node_standby, 'wal_level_', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 1, \$stdout, \$stderr);
@@ -858,7 +859,7 @@ check_pg_recvlogical_stderr($handle,
 drop_logical_slots('wal_level_');
 
 # create the logical slots
-create_logical_slots($node_standby, 'drop_db_');
+create_logical_slots($node_standby, 'drop_db_', 1);
 
 $handle = make_slot_active($node_standby, 'drop_db_', 1, \$stdout, \$stderr);
 
@@ -922,14 +923,14 @@ $node_cascading_standby->append_conf(
 $node_cascading_standby->start;
 
 # create the logical slots
-create_logical_slots($node_standby, 'promotion_');
+create_logical_slots($node_standby, 'promotion_', 1);
 
 # Wait for the cascading standby to catchup before creating the slots
 $node_standby->wait_for_replay_catchup($node_cascading_standby,
 	$node_primary);
 
 # create the logical slots on the cascading standby too
-create_logical_slots($node_cascading_standby, 'promotion_');
+create_logical_slots($node_cascading_standby, 'promotion_', 1);
 
 # Make slots actives
 $handle =
-- 
2.43.5



  [application/octet-stream] v5-PG17-2-0001-Stabilize-035_standby_logical_decoding.pl.patch (7.3K, ../../OSCPR01MB1496631866E979BD510BC5448F5AF2@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-v5-PG17-2-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From ea258ce0f4d2f416f0d33af974abe87219458511 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v5-PG17-2] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 66 +++++++++----------
 1 file changed, 31 insertions(+), 35 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..2b7ae4b74e9 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,9 +212,8 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
 
 	# message should be issued
@@ -226,18 +222,24 @@ sub check_for_invalidation
 			$log_start),
 		"inactiveslot slot invalidation is logged $test_name");
 
-	ok( $node_standby->log_contains(
-			"invalidating obsolete replication slot \"$active_slot\"",
-			$log_start),
-		"activeslot slot invalidation is logged $test_name");
-
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		ok( $node_standby->log_contains(
+				"invalidating obsolete replication slot \"$active_slot\"",
+				$log_start),
+			"activeslot slot invalidation is logged $test_name");
+
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +252,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that replication slots are not activated
+# for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -550,10 +553,6 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,19 +561,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
 $handle =
   make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
 
@@ -651,7 +642,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -687,7 +678,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -711,6 +702,11 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# This scenario won't produce the race condition by a xl_running_xacts, so
+# activate the slot. See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
+	\$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -779,7 +775,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -823,7 +819,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-02 15:00  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-02 15:00 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

On Wed, Apr 02, 2025 at 12:13:52PM +0000, Hayato Kuroda (Fujitsu) wrote:
> Dear Amit, Bertrand,
> 
> > The other idea to simplify the changes for backbranches:
> > sub reactive_slots_change_hfs_and_wait_for_xmins
> > {
> > ...
> > +  my ($slot_prefix, $hsf, $invalidated, $needs_active_slot) = @_;
> > 
> >   # create the logical slots
> > -  create_logical_slots($node_standby, $slot_prefix);
> > +  create_logical_slots($node_standby, $slot_prefix, $needs_active_slot);
> > 
> > ...
> > +  if ($needs_active_slot)
> > +  {
> > +    $handle =
> > +      make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
> > +  }
> > 
> > What if this function doesn't take input parameter needs_active_slot
> > and rather removes the call to make_slot_active? We will call
> > make_slot_active only at the required places. This should make the
> > changes much less because after that, we don't need to make changes
> > related to drop and create. Sure, in some cases, we will test two
> > inactive slots instead of one, but I guess that would be the price to
> > keep the tests simple and more like HEAD.
> 
> Actually, I could not decide which one is better, so let me share both drafts.

Thanks!

> V5-PG17-1 uses the previous approach, and v5-PG17-2 uses new proposed one.
> Bertrand, which one do you like?

I do prefer v5-PG17-2 as it is "closer" to HEAD. That said, I think that we should
keep the slots active and only avoid doing the checks for them (they are invalidated
that's fine, they are not that's fine too).

Also I think that we should change this part:

"
 # Verify that invalidated logical slots do not lead to retaining WAL.
@@ -602,7 +610,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
        'postgres',
        "SELECT restart_lsn FROM pg_replication_slots
-               WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+               WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );

" to be on the safe side of thing.

What do you think of the attached (to apply on top of v5-PG17-2)?

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 2b7ae4b74e9..d1be179fed6 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -171,24 +171,28 @@ sub change_hot_standby_feedback_and_wait_for_xmins
 # Check reason for conflict in pg_replication_slots.
 sub check_slots_conflict_reason
 {
-	my ($slot_prefix, $reason) = @_;
+	my ($slot_prefix, $reason, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
-
-	$res = $node_standby->safe_psql(
-		'postgres', qq(
-			 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
-	);
-
-	is($res, "$reason", "$active_slot reason for conflict is $reason");
-
 	$res = $node_standby->safe_psql(
 		'postgres', qq(
 			 select invalidation_reason from pg_replication_slots where slot_name = '$inactive_slot' and conflicting;)
 	);
 
 	is($res, "$reason", "$inactive_slot reason for conflict is $reason");
+
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$res = $node_standby->safe_psql(
+			'postgres', qq(
+				 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
+		);
+
+		is($res, "$reason", "$active_slot reason for conflict is $reason");
+
+	}
 }
 
 # Drop the slots, re-create them, change hot_standby_feedback,
@@ -205,6 +209,9 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
+	$handle =
+	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -252,8 +259,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon. We ensure that replication slots are not activated
-# for tests that might produce this race condition though.
+# the catalog xmin horizon. We ensure to not test for invalidations in such
+# cases.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -553,6 +560,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
+$node_standby->poll_query_until('testdb',
+	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
+) or die "replication slot stats of vacuum_full_activeslot not updated";
+
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -564,16 +575,19 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('vacuum_full_', 'rows_removed');
+check_slots_conflict_reason('vacuum_full_', 'rows_removed', 0);
+
+# Ensure that replication slot stats are not removed after invalidation.
+is( $node_standby->safe_psql(
+		'testdb',
+		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
+	),
+	't',
+	'replication slot stats not removed after invalidation');
 
 $handle =
   make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -583,7 +597,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $node_standby->restart;
 
 # Verify reason for conflict is retained across a restart.
-check_slots_conflict_reason('vacuum_full_', 'rows_removed');
+check_slots_conflict_reason('vacuum_full_', 'rows_removed', 0);
 
 ##################################################
 # Verify that invalidated logical slots do not lead to retaining WAL.
@@ -593,7 +607,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
 	'postgres',
 	"SELECT restart_lsn FROM pg_replication_slots
-		WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+		WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );
 
 chomp($restart_lsn);
@@ -645,16 +659,11 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('row_removal_', 'rows_removed');
+check_slots_conflict_reason('row_removal_', 'rows_removed', 0);
 
 $handle =
   make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
@@ -681,16 +690,11 @@ check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
+check_slots_conflict_reason('shared_row_removal_', 'rows_removed', 0);
 
 $handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
 	\$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
@@ -702,11 +706,6 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
-# This scenario won't produce the race condition by a xl_running_xacts, so
-# activate the slot. See comments atop wait_until_vacuum_can_remove().
-make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
-	\$stderr);
-
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -778,14 +777,10 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('pruning_', 'rows_removed');
+check_slots_conflict_reason('pruning_', 'rows_removed', 0);
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -822,7 +817,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
-check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
+check_slots_conflict_reason('wal_level_', 'wal_level_insufficient', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 0, \$stdout, \$stderr);


Attachments:

  [text/plain] v5-PG17-2-0001-bertrand.patch.txt (7.9K, ../../Z+1RJElgG%[email protected]/2-v5-PG17-2-0001-bertrand.patch.txt)
  download | inline diff:
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 2b7ae4b74e9..d1be179fed6 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -171,24 +171,28 @@ sub change_hot_standby_feedback_and_wait_for_xmins
 # Check reason for conflict in pg_replication_slots.
 sub check_slots_conflict_reason
 {
-	my ($slot_prefix, $reason) = @_;
+	my ($slot_prefix, $reason, $checks_active_slot) = @_;
 
-	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
-
-	$res = $node_standby->safe_psql(
-		'postgres', qq(
-			 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
-	);
-
-	is($res, "$reason", "$active_slot reason for conflict is $reason");
-
 	$res = $node_standby->safe_psql(
 		'postgres', qq(
 			 select invalidation_reason from pg_replication_slots where slot_name = '$inactive_slot' and conflicting;)
 	);
 
 	is($res, "$reason", "$inactive_slot reason for conflict is $reason");
+
+	if ($checks_active_slot)
+	{
+		my $active_slot = $slot_prefix . 'activeslot';
+
+		$res = $node_standby->safe_psql(
+			'postgres', qq(
+				 select invalidation_reason from pg_replication_slots where slot_name = '$active_slot' and conflicting;)
+		);
+
+		is($res, "$reason", "$active_slot reason for conflict is $reason");
+
+	}
 }
 
 # Drop the slots, re-create them, change hot_standby_feedback,
@@ -205,6 +209,9 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
+	$handle =
+	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
+
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -252,8 +259,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon. We ensure that replication slots are not activated
-# for tests that might produce this race condition though.
+# the catalog xmin horizon. We ensure to not test for invalidations in such
+# cases.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -553,6 +560,10 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
+$node_standby->poll_query_until('testdb',
+	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
+) or die "replication slot stats of vacuum_full_activeslot not updated";
+
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -564,16 +575,19 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('vacuum_full_', 'rows_removed');
+check_slots_conflict_reason('vacuum_full_', 'rows_removed', 0);
+
+# Ensure that replication slot stats are not removed after invalidation.
+is( $node_standby->safe_psql(
+		'testdb',
+		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
+	),
+	't',
+	'replication slot stats not removed after invalidation');
 
 $handle =
   make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"vacuum_full_activeslot\""
-);
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -583,7 +597,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 $node_standby->restart;
 
 # Verify reason for conflict is retained across a restart.
-check_slots_conflict_reason('vacuum_full_', 'rows_removed');
+check_slots_conflict_reason('vacuum_full_', 'rows_removed', 0);
 
 ##################################################
 # Verify that invalidated logical slots do not lead to retaining WAL.
@@ -593,7 +607,7 @@ check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 my $restart_lsn = $node_standby->safe_psql(
 	'postgres',
 	"SELECT restart_lsn FROM pg_replication_slots
-		WHERE slot_name = 'vacuum_full_activeslot' AND conflicting;"
+		WHERE slot_name = 'vacuum_full_inactiveslot' AND conflicting;"
 );
 
 chomp($restart_lsn);
@@ -645,16 +659,11 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('row_removal_', 'rows_removed');
+check_slots_conflict_reason('row_removal_', 'rows_removed', 0);
 
 $handle =
   make_slot_active($node_standby, 'row_removal_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a shared catalog table
 # Scenario 3: conflict due to row removal with hot_standby_feedback off.
@@ -681,16 +690,11 @@ check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
+check_slots_conflict_reason('shared_row_removal_', 'rows_removed', 0);
 
 $handle = make_slot_active($node_standby, 'shared_row_removal_', 0, \$stdout,
 	\$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"shared_row_removal_activeslot\""
-);
-
 ##################################################
 # Recovery conflict: Same as Scenario 2 but on a non catalog table
 # Scenario 4: No conflict expected.
@@ -702,11 +706,6 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
-# This scenario won't produce the race condition by a xl_running_xacts, so
-# activate the slot. See comments atop wait_until_vacuum_can_remove().
-make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
-	\$stderr);
-
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -778,14 +777,10 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
-check_slots_conflict_reason('pruning_', 'rows_removed');
+check_slots_conflict_reason('pruning_', 'rows_removed', 0);
 
 $handle = make_slot_active($node_standby, 'pruning_', 0, \$stdout, \$stderr);
 
-# We are not able to read from the slot as it has been invalidated
-check_pg_recvlogical_stderr($handle,
-	"can no longer get changes from replication slot \"pruning_activeslot\"");
-
 # Turn hot_standby_feedback back on
 change_hot_standby_feedback_and_wait_for_xmins(1, 1);
 
@@ -822,7 +817,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
-check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
+check_slots_conflict_reason('wal_level_', 'wal_level_insufficient', 1);
 
 $handle =
   make_slot_active($node_standby, 'wal_level_', 0, \$stdout, \$stderr);


^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-03 05:34  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 2 replies; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-03 05:34 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; Bertrand Drouvot <[email protected]>; +Cc: [email protected] <[email protected]>

Dear Bertrand, Amit,

> > I do prefer v5-PG17-2 as it is "closer" to HEAD. That said, I think that we should
> > keep the slots active and only avoid doing the checks for them (they are
> invalidated
> > that's fine, they are not that's fine too).
> >
> 
> I don't mind doing that, but there is no benefit in making slots
> active unless we can validate them. And we will end up adding some
> more checks, as in function check_slots_conflict_reason without any
> advantage. I feel Kuroda-San's second patch is simple, and we have
> fewer chances to make mistakes and easy to maintain in the future as
> well.

I have concerns for Bertrand's patch that it could introduce another timing
issue. E.g., if the activated slots are not invalidated, dropping slots is keep
being activated so the dropping might be fail. I did not reproduce this but
something like this can happen if we activate slots.

Attached patch has a conclusion of these discussions, slots are created but
it seldomly be activated.

Naming of patches are bit different, but please ignore...

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] v5-PG16-0001-Stabilize-035_standby_logical_decoding.pl.patch (5.8K, ../../OSCPR01MB14966608F78B5BFC907CF8E82F5AE2@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-v5-PG16-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From c69b5b2d0b53c28ac99705b2c1507be24658104b Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v5-PG16] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 41 +++++++++++--------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8120dfc2132..1cf58f453f5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,7 +212,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -231,13 +228,17 @@ sub check_for_invalidation
 			$log_start),
 		"activeslot slot invalidation is logged $test_name");
 
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +251,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that replication slots are not activated
+# for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -550,7 +552,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -632,7 +634,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -668,7 +670,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -692,6 +694,11 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# This scenario won't produce the race condition by a xl_running_xacts, so
+# activate the slot. See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
+	\$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -754,7 +761,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -798,7 +805,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
-- 
2.43.5



  [application/octet-stream] v5-PG17-0001-Stabilize-035_standby_logical_decoding.pl.patch (6.9K, ../../OSCPR01MB14966608F78B5BFC907CF8E82F5AE2@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-v5-PG17-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From 5dce899cb4856908ea41a8817fa71135b505c0c2 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v5-PG17] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip activating slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 53 +++++++++----------
 1 file changed, 24 insertions(+), 29 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..752e31960ea 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,7 +212,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -231,13 +228,17 @@ sub check_for_invalidation
 			$log_start),
 		"activeslot slot invalidation is logged $test_name");
 
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +251,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that replication slots are not activated
+# for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -550,10 +552,6 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,19 +560,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
 $handle =
   make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
 
@@ -651,7 +641,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -687,7 +677,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -711,6 +701,11 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# This scenario won't produce the race condition by a xl_running_xacts, so
+# activate the slot. See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
+	\$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -779,7 +774,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -823,7 +818,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.43.5



  [application/octet-stream] 0001-Fix-invalid-referring-of-hash-ref-for-replication-sl.patch (2.4K, ../../OSCPR01MB14966608F78B5BFC907CF8E82F5AE2@OSCPR01MB14966.jpnprd01.prod.outlook.com/4-0001-Fix-invalid-referring-of-hash-ref-for-replication-sl.patch)
  download | inline diff:
From 47249c139c7fe7671d02657c6fb5f9bed128af14 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Thu, 3 Apr 2025 12:12:12 +0900
Subject: [PATCH] Fix invalid referring of hash-ref for replication slots

hash-ref gerenated by slot() did not have key 'slot_name', but some codes
referred it. Fix it by referring 'plugin' instead.
---
 src/test/recovery/t/006_logical_decoding.pl           | 8 ++++----
 src/test/recovery/t/010_logical_decoding_timelines.pl | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/test/recovery/t/006_logical_decoding.pl b/src/test/recovery/t/006_logical_decoding.pl
index a5678bc4dc4..2137c4e5e30 100644
--- a/src/test/recovery/t/006_logical_decoding.pl
+++ b/src/test/recovery/t/006_logical_decoding.pl
@@ -161,8 +161,8 @@ SKIP:
 	is($node_primary->psql('postgres', 'DROP DATABASE otherdb'),
 		3, 'dropping a DB with active logical slots fails');
 	$pg_recvlogical->kill_kill;
-	is($node_primary->slot('otherdb_slot')->{'slot_name'},
-		undef, 'logical slot still exists');
+	is($node_primary->slot('otherdb_slot')->{'plugin'},
+		'test_decoding', 'logical slot still exists');
 }
 
 $node_primary->poll_query_until('otherdb',
@@ -171,8 +171,8 @@ $node_primary->poll_query_until('otherdb',
 
 is($node_primary->psql('postgres', 'DROP DATABASE otherdb'),
 	0, 'dropping a DB with inactive logical slots succeeds');
-is($node_primary->slot('otherdb_slot')->{'slot_name'},
-	undef, 'logical slot was actually dropped with DB');
+is($node_primary->slot('otherdb_slot')->{'plugin'},
+	'', 'logical slot was actually dropped with DB');
 
 # Test logical slot advancing and its durability.
 # Passing failover=true (last arg) should not have any impact on advancing.
diff --git a/src/test/recovery/t/010_logical_decoding_timelines.pl b/src/test/recovery/t/010_logical_decoding_timelines.pl
index 08615f1fca8..0199ae95abf 100644
--- a/src/test/recovery/t/010_logical_decoding_timelines.pl
+++ b/src/test/recovery/t/010_logical_decoding_timelines.pl
@@ -94,8 +94,8 @@ is( $node_replica->safe_psql(
 		'postgres', q[SELECT 1 FROM pg_database WHERE datname = 'dropme']),
 	'',
 	'dropped DB dropme on standby');
-is($node_primary->slot('dropme_slot')->{'slot_name'},
-	undef, 'logical slot was actually dropped on standby');
+is($node_primary->slot('dropme_slot')->{'plugin'},
+	'', 'logical slot was actually dropped on standby');
 
 # Back to testing failover...
 $node_primary->safe_psql('postgres',
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-03 05:58  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  1 sibling, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-03 05:58 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; [email protected] <[email protected]>

>  Isn't patch 0001-Fix-invalid-referring-of-hash-ref-for-replication-sl
> unrelated to this thread? Or am, I missing something?

I did attach wrongly, PSA correct set. Sorry for inconvenience.

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] v5-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch (7.7K, ../../OSCPR01MB149666E4ECE1C38C60E496EDCF5AE2@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-v5-0001-Stabilize-035_standby_logical_decoding.pl-by-usin.patch)
  download | inline diff:
From fbb3658c17dbf5bd4fdcee1803fda6a40d3839a4 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 14:19:50 +0900
Subject: [PATCH v5] Stabilize 035_standby_logical_decoding.pl by using the
 injection_points.

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip generating the record when the instance attached to a new
injection point.

This failure can happen since logical decoding is allowed on the standby server.
But the interface of injection_points we used exists only on master, so we do
not backpatch.
---
 src/backend/storage/ipc/standby.c             | 12 ++++
 .../t/035_standby_logical_decoding.pl         | 56 ++++++++++++++-----
 2 files changed, 54 insertions(+), 14 deletions(-)

diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c
index 5acb4508f85..7fa8d9247e0 100644
--- a/src/backend/storage/ipc/standby.c
+++ b/src/backend/storage/ipc/standby.c
@@ -31,6 +31,7 @@
 #include "storage/sinvaladt.h"
 #include "storage/standby.h"
 #include "utils/hsearch.h"
+#include "utils/injection_point.h"
 #include "utils/ps_status.h"
 #include "utils/timeout.h"
 #include "utils/timestamp.h"
@@ -1287,6 +1288,17 @@ LogStandbySnapshot(void)
 
 	Assert(XLogStandbyInfoActive());
 
+#ifdef USE_INJECTION_POINTS
+	if (IS_INJECTION_POINT_ATTACHED("skip-log-running-xacts"))
+	{
+		/*
+		 * This record could move slot's xmin forward during decoding, leading
+		 * to unpredictable results, so skip it when requested by the test.
+		 */
+		return GetInsertRecPtr();
+	}
+#endif
+
 	/*
 	 * Get details of any AccessExclusiveLocks being held at the moment.
 	 */
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index c31cab06f1c..52ebd24f7f1 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -10,6 +10,11 @@ use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
 
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
 my ($stdout, $stderr, $cascading_stdout, $cascading_stderr, $handle);
 
 my $node_primary = PostgreSQL::Test::Cluster->new('primary');
@@ -241,16 +246,19 @@ sub check_for_invalidation
 # VACUUM command, $sql the sql to launch before triggering the vacuum and
 # $to_vac the relation to vacuum.
 #
-# Note that pg_current_snapshot() is used to get the horizon.  It does
-# not generate a Transaction/COMMIT WAL record, decreasing the risk of
-# seeing a xl_running_xacts that would advance an active replication slot's
-# catalog_xmin.  Advancing the active replication slot's catalog_xmin
-# would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# Note that the injection_point avoids seeing a xl_running_xacts that could
+# advance an active replication slot's catalog_xmin. Advancing the active
+# replication slot's catalog_xmin would break some tests that expect the
+# active slot to conflict with the catalog xmin horizon.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
 
+	# Note that from this point the checkpointer and bgwriter will skip writing
+	# xl_running_xacts record.
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_attach('skip-log-running-xacts', 'error');");
+
 	# Get the current xid horizon,
 	my $xid_horizon = $node_primary->safe_psql('testdb',
 		qq[select pg_snapshot_xmin(pg_current_snapshot());]);
@@ -268,6 +276,12 @@ sub wait_until_vacuum_can_remove
 	$node_primary->safe_psql(
 		'testdb', qq[VACUUM $vac_option verbose $to_vac;
 										  INSERT INTO flush_wal DEFAULT VALUES;]);
+
+	$node_primary->wait_for_replay_catchup($node_standby);
+
+	# Resume generating the xl_running_xacts record
+	$node_primary->safe_psql('testdb',
+		"SELECT injection_points_detach('skip-log-running-xacts');");
 }
 
 ########################
@@ -285,6 +299,14 @@ autovacuum = off
 $node_primary->dump_info;
 $node_primary->start;
 
+# Check if the extension injection_points is available, as it may be
+# possible that this script is run with installcheck, where the module
+# would not be installed by default.
+if (!$node_primary->check_extension('injection_points'))
+{
+	plan skip_all => 'Extension injection_points not installed';
+}
+
 $node_primary->psql('postgres', q[CREATE DATABASE testdb]);
 
 $node_primary->safe_psql('testdb',
@@ -528,6 +550,9 @@ is($result, qq(10), 'check replicated inserts after subscription on standby');
 $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
+# Create the injection_points extension
+$node_primary->safe_psql('testdb', 'CREATE EXTENSION injection_points;');
+
 ##################################################
 # Recovery conflict: Invalidate conflicting slots, including in-use slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
@@ -557,8 +582,6 @@ wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
 								 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
 
@@ -656,8 +679,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
 							 DROP TABLE conflict_test;', 'pg_class');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
 
@@ -690,8 +711,6 @@ wait_until_vacuum_can_remove(
 	'', 'CREATE ROLE create_trash;
 							 DROP ROLE create_trash;', 'pg_authid');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
 	'with vacuum on pg_authid');
@@ -724,8 +743,6 @@ wait_until_vacuum_can_remove(
 							 INSERT INTO conflict_test(x,y) SELECT s, s::text FROM generate_series(1,4) s;
 							 UPDATE conflict_test set x=1, y=1;', 'conflict_test');
 
-$node_primary->wait_for_replay_catchup($node_standby);
-
 # message should not be issued
 ok( !$node_standby->log_contains(
 		"invalidating obsolete slot \"no_conflict_inactiveslot\"", $logstart),
@@ -773,6 +790,13 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('no_conflict_', 'pruning_', 0,
 	0);
 
+# Injection_point avoids seeing a xl_running_xacts. This is required because if
+# it is generated between the last two updates, then the catalog_xmin of the
+# active slot could be updated, and hence, the conflict won't occur. See
+# comments atop wait_until_vacuum_can_remove.
+$node_primary->safe_psql('testdb',
+	"SELECT injection_points_attach('skip-log-running-xacts', 'error');");
+
 # This should trigger the conflict
 $node_primary->safe_psql('testdb',
 	qq[CREATE TABLE prun(id integer, s char(2000)) WITH (fillfactor = 75, user_catalog_table = true);]
@@ -785,6 +809,10 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 
 $node_primary->wait_for_replay_catchup($node_standby);
 
+# Resume generating the xl_running_xacts record
+$node_primary->safe_psql('testdb',
+	"SELECT injection_points_detach('skip-log-running-xacts');");
+
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
 
-- 
2.43.5



  [application/octet-stream] v5-PG16-0001-Stabilize-035_standby_logical_decoding.pl.patch (5.8K, ../../OSCPR01MB149666E4ECE1C38C60E496EDCF5AE2@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-v5-PG16-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From c69b5b2d0b53c28ac99705b2c1507be24658104b Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v5-PG16] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip using the active slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 41 +++++++++++--------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 8120dfc2132..1cf58f453f5 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,7 +212,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -231,13 +228,17 @@ sub check_for_invalidation
 			$log_start),
 		"activeslot slot invalidation is logged $test_name");
 
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +251,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that replication slots are not activated
+# for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -550,7 +552,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -632,7 +634,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -668,7 +670,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -692,6 +694,11 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# This scenario won't produce the race condition by a xl_running_xacts, so
+# activate the slot. See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
+	\$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -754,7 +761,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -798,7 +805,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
-- 
2.43.5



  [application/octet-stream] v5-PG17-0001-Stabilize-035_standby_logical_decoding.pl.patch (6.9K, ../../OSCPR01MB149666E4ECE1C38C60E496EDCF5AE2@OSCPR01MB14966.jpnprd01.prod.outlook.com/4-v5-PG17-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From 5dce899cb4856908ea41a8817fa71135b505c0c2 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <[email protected]>
Date: Wed, 26 Mar 2025 19:03:50 +0900
Subject: [PATCH v5-PG17] Stabilize 035_standby_logical_decoding.pl

This test tries to invalidate slots on standby server, by running VACUUM on
primary and discarding needed tuples for slots. The problem is that
xl_running_xacts records are sotimetimes generated while testing, it advances
the catalog_xmin so that the invalidation might not happen in some cases.

The fix is to skip activating slots for some testcases.
---
 .../t/035_standby_logical_decoding.pl         | 53 +++++++++----------
 1 file changed, 24 insertions(+), 29 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index aeb79f51e71..752e31960ea 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,7 +212,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -231,13 +228,17 @@ sub check_for_invalidation
 			$log_start),
 		"activeslot slot invalidation is logged $test_name");
 
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +251,8 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon. We ensure that replication slots are not activated
+# for tests that might produce this race condition though.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -550,10 +552,6 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,19 +560,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
 $handle =
   make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
 
@@ -651,7 +641,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -687,7 +677,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -711,6 +701,11 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# This scenario won't produce the race condition by a xl_running_xacts, so
+# activate the slot. See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout,
+	\$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -779,7 +774,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -823,7 +818,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-03 06:59  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  1 sibling, 2 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-03 06:59 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi,

On Thu, Apr 03, 2025 at 05:34:10AM +0000, Hayato Kuroda (Fujitsu) wrote:
> Dear Bertrand, Amit,
> 
> > > I do prefer v5-PG17-2 as it is "closer" to HEAD. That said, I think that we should
> > > keep the slots active and only avoid doing the checks for them (they are
> > invalidated
> > > that's fine, they are not that's fine too).
> > >
> > 
> > I don't mind doing that, but there is no benefit in making slots
> > active unless we can validate them. And we will end up adding some
> > more checks, as in function check_slots_conflict_reason without any
> > advantage.

I think that there is advantage. The pros are:

- the test would be closer to HEAD from a behavioural point of view
- it's very rare to hit the corner cases: so the test would behave the same
as on HEAD most of the time (and when it does not that would not hurt as the
checks are nor done)
- Kuroda-San's patch removes "or die "replication slot stats of vacuum_full_activeslot not updated"
while keeping the slot active is able to keep it (should the slot being invalidated
or not). But more on that in the comment === 1 below.

> I feel Kuroda-San's second patch is simple, and we have
> > fewer chances to make mistakes and easy to maintain in the future as
> > well.

Yeah maybe but the price to pay is to discard the pros above. That said, I'm also
fine with Kuroda-San's patch if both of you feel that it's better.

> I have concerns for Bertrand's patch that it could introduce another timing
> issue. E.g., if the activated slots are not invalidated, dropping slots is keep
> being activated so the dropping might be fail.

Yeah, but the drop is done with "$node_standby->psql" so that the test does not
produce an error. It would produce an error should we use "$node_standby->safe_psql"
instead.

> I did not reproduce this but
> something like this can happen if we activate slots.

You can see it that way (+ reproducer.txt):

"
+       my $bdt = $node_standby->safe_psql('postgres', qq[SELECT * from pg_replication_slots]);
+       note "BDT: $bdt";
+
        $node_standby->psql('postgres',
                qq[SELECT pg_drop_replication_slot('$inactive_slot')]);
"

You'd see the slot being active and the "$node_standby->psql" not reporting
any error.

> Attached patch has a conclusion of these discussions, slots are created but
> it seldomly be activated.

Thanks for the patch!

=== 1

-$node_standby->poll_query_until('testdb',
-       qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(

I wonder if we could not keep this test and make the slot active for the
vacuum full case. Looking at drongo's failure in [1], there is no occurence
of "vacuum full" and that's probably linked to Andres's explanation in [2]:

"
a VACUUM FULL on pg_class is
used, which prevents logical decoding from progressing after it started (due
to the logged AEL at the start of VACFULL).
"

meaning that the active slot is invalidated even if the catalog xmin's moves
forward due to xl_running_xacts.

[1]: https://www.postgresql.org/message-id/386386.1737736935%40sss.pgh.pa.us
[2]: https://www.postgresql.org/message-id/zqypkuvtihtd2zbmwdfmcceujg4fuakrhojmjkxpp7t4udqkty%40couhenc7d...

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-07 06:15  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-07 06:15 UTC (permalink / raw)
  To: 'Bertrand Drouvot' <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Dear Bertrand,

> I wonder if we could not keep this test and make the slot active for the
> vacuum full case. Looking at drongo's failure in [1], there is no occurence
> of "vacuum full" and that's probably linked to Andres's explanation in [2]:
> 
> "
> a VACUUM FULL on pg_class is
> used, which prevents logical decoding from progressing after it started (due
> to the logged AEL at the start of VACFULL).
> "
> 

I had been debugging and found the case that VACUUM FULL also has a timing issue.
This means the we cannot keep the testcase.

PSA the reproducer for PG17. IIUC this can happen even in PG16.
I considered what happened here;

1. Run a CHECKPOINT and wait sometime in wait_until_vacuum_can_remove().
   This ensures that RUNNING_XACTS record can be generated and catalog_xmin can
   be advanced after the user SQLs.
2. Assuming that another RUNNING_XACTS record is generated *WHILE* doing a VACUUM
   FULL. This can be done by the periodic checkpoint or the reproducer.
3. Logical walsender detects the RUNNING_XACTS record.
   Note that this must be done before startup tries to invalidate slot.
4. In sometime the walsender receives the ack and advance the catalog_xmin.
   Note again that this must be done before startup tries to invalidate slot.
5. Startup process detects the PRUNE_ON_ACCESS record and tries to invalidate the
   slot. However, the catalog_xmin has been advanced so that the invalidation
   cannot be done.

Analysis
========
While analyzing this workload, I found that VACUUM FULL can generate four
PRUNE_ON_ACCESS records. More especially, first two records are generated while
clustering the table, others are done while updating pg_database.datfrozenxid.
Interestingly, latter records are genareted after the transaction is finished;
the VACUUM FULL command itselfs ends up the txn once (in vacuum_rel) and then
continue working on. Without the delay in testcode, the first PRUNE record leads
the invalidation the slot, and with the delay fourth PRUNE leads it. Per my
analysis, snapshotConflictHorizon is the xid which first PRUNE records exist.

Based on the fact, I considered that catalog_xmin can be advanced till the between
(non-)transactional PRUNE records. RequestCheckpoint() is added to generate the
RUNNING_XACTS in-between them.

Very thanks Amit for supporting me off-list for reproducing the issue.

Best regards,
Hayato Kuroda
Fujitsu LIMITED

Attachments:

  [application/octet-stream] repro_pg17.diffs (2.2K, ../../OSCPR01MB14966A5BBB6A16357B1D49D9CF5AA2@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-repro_pg17.diffs)
  download

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-07 09:10  Bertrand Drouvot <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-07 09:10 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi Kuroda-san,

On Mon, Apr 07, 2025 at 06:15:13AM +0000, Hayato Kuroda (Fujitsu) wrote:
> I had been debugging and found the case that VACUUM FULL also has a timing issue.
> This means the we cannot keep the testcase.
> 
> PSA the reproducer for PG17. IIUC this can happen even in PG16.
> I considered what happened here;
> 
> 1. Run a CHECKPOINT and wait sometime in wait_until_vacuum_can_remove().
>    This ensures that RUNNING_XACTS record can be generated and catalog_xmin can
>    be advanced after the user SQLs.
> 2. Assuming that another RUNNING_XACTS record is generated *WHILE* doing a VACUUM
>    FULL. This can be done by the periodic checkpoint or the reproducer.
> 3. Logical walsender detects the RUNNING_XACTS record.
>    Note that this must be done before startup tries to invalidate slot.
> 4. In sometime the walsender receives the ack and advance the catalog_xmin.
>    Note again that this must be done before startup tries to invalidate slot.
> 5. Startup process detects the PRUNE_ON_ACCESS record and tries to invalidate the
>    slot. However, the catalog_xmin has been advanced so that the invalidation
>    cannot be done.

Thanks for the testing and explanation! I did apply your repro and I'm able to
see the test failing (with an active slot). The scenario is more unlikely
to happen (as compare to the non vacuum full cases) and that's why it was not
visible in drongo's reports in [1]. So yeah, let's do as you suggested and do
not make the slot active for the vacuum full case too.

[1]: https://www.postgresql.org/message-id/[email protected]

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-07 09:58  Bertrand Drouvot <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  1 sibling, 0 replies; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-07 09:58 UTC (permalink / raw)
  To: Amit Kapila <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; [email protected] <[email protected]>

Hi,

On Mon, Apr 07, 2025 at 03:16:07PM +0530, Amit Kapila wrote:
> On Thu, Apr 3, 2025 at 3:15 PM Amit Kapila <[email protected]> wrote:
> > Hmm, but adding some additional smarts also makes this test less easy
> > to backpatch. I see your points related to the benefits, but I still
> > mildly prefer to go with the lesser changes approach for backbranches
> > patch. Normally, we don't enhance backbranches code without making
> > equivalent changes in HEAD, so adding some new bugs only in
> > backbranches has a lesser chance.
> >
> 
> Bertrand, do you agree with the fewer changes approach (where active
> slots won't be tested) for backbranches? I think now that we have
> established that the vacuum full test is also prone to failure due to
> race condition in the test, this is the only remaining open point.

Yeah that's all good on my side, let's keep it that way and don't make the slot
active.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* RE: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-08 02:00  Hayato Kuroda (Fujitsu) <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Hayato Kuroda (Fujitsu) @ 2025-04-08 02:00 UTC (permalink / raw)
  To: 'Amit Kapila' <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; [email protected] <[email protected]>

Dear Amit,

> 
> I have changed quite a few comments and commit message for the PG17
> patch in the attached. Can you update PG16 patch based on this and
> also use the same commit message as used in attached for all the three
> patches?

Your patch looks good to me and it could pass on my env. PSA patches for PG16.
Patch for PG17 is not changed, just renamed.

Best regards,
Hayato Kuroda
FUJITSU LIMITED



Attachments:

  [application/octet-stream] v6-PG17-0001-Stabilize-035_standby_logical_decoding.pl.patch (8.7K, ../../OSCPR01MB14966BE900F2F29594D5E1FE7F5B52@OSCPR01MB14966.jpnprd01.prod.outlook.com/2-v6-PG17-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From 7ae736c680cb20432d1eaed1a6bb033c21675253 Mon Sep 17 00:00:00 2001
From: Amit Kapila <[email protected]>
Date: Mon, 7 Apr 2025 17:58:23 +0530
Subject: [PATCH v6-PG17] Stabilize 035_standby_logical_decoding.pl.

Some tests try to invalidate logical slots on the standby server by
running VACUUM on the primary. The problem is that xl_running_xacts was
getting generated and replayed before the VACUUM command, leading to the
advancement of the active slot's catalog_xmin. Due to this, active slots
were not getting invalidated, leading to test failures.

We fix it by skipping the generation of xl_running_xacts for the required
tests with the help of injection points. As the required interface for
injection points was not present in back branches, we fixed the failing
tests in them by disallowing the slot to become active for the required
cases (where rows_removed conflict could be generated).
---
 .../t/035_standby_logical_decoding.pl         | 64 +++++++++----------
 1 file changed, 29 insertions(+), 35 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 4eca17885d6..58c4402e80e 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,7 +212,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -231,13 +228,17 @@ sub check_for_invalidation
 			$log_start),
 		"activeslot slot invalidation is logged $test_name");
 
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +251,11 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon.  Even with the above precaution, there is a risk
+# of xl_running_xacts record being logged and replayed before the VACUUM
+# command, leading to the test failure.  So, we ensured that replication slots
+# are not activated for tests that can invalidate slots due to 'rows_removed'
+# conflict reason.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -532,11 +537,8 @@ $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
 ##################################################
-# Recovery conflict: Invalidate conflicting slots, including in-use slots
+# Recovery conflict: Invalidate conflicting slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
-#
-# In passing, ensure that replication slot stats are not removed when the
-# active slot is invalidated.
 ##################################################
 
 # One way to produce recovery conflict is to create/drop a relation and
@@ -550,10 +552,6 @@ reactive_slots_change_hfs_and_wait_for_xmins('behaves_ok_', 'vacuum_full_',
 $node_primary->safe_psql('testdb',
 	qq[INSERT INTO decoding_test(x,y) SELECT 100,'100';]);
 
-$node_standby->poll_query_until('testdb',
-	qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-) or die "replication slot stats of vacuum_full_activeslot not updated";
-
 # This should trigger the conflict
 wait_until_vacuum_can_remove(
 	'full', 'CREATE TABLE conflict_test(x integer, y text);
@@ -562,19 +560,11 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('vacuum_full_', 'rows_removed');
 
-# Ensure that replication slot stats are not removed after invalidation.
-is( $node_standby->safe_psql(
-		'testdb',
-		qq[SELECT total_txns > 0 FROM pg_stat_replication_slots WHERE slot_name = 'vacuum_full_activeslot']
-	),
-	't',
-	'replication slot stats not removed after invalidation');
-
 $handle =
   make_slot_active($node_standby, 'vacuum_full_', 0, \$stdout, \$stderr);
 
@@ -639,7 +629,7 @@ ok(!-f "$standby_walfile",
 	"invalidated logical slots do not lead to retaining WAL");
 
 ##################################################
-# Recovery conflict: Invalidate conflicting slots, including in-use slots
+# Recovery conflict: Invalidate conflicting slots
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
@@ -660,7 +650,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('row_removal_', 'rows_removed');
@@ -696,7 +686,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('shared_row_removal_', 'rows_removed');
@@ -720,6 +710,10 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# As this scenario is not expected to produce any conflict, so activate the slot.
+# See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout, \$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -763,7 +757,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 0);
 $node_standby->restart;
 
 ##################################################
-# Recovery conflict: Invalidate conflicting slots, including in-use slots
+# Recovery conflict: Invalidate conflicting slots
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
@@ -788,7 +782,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify reason for conflict is 'rows_removed' in pg_replication_slots
 check_slots_conflict_reason('pruning_', 'rows_removed');
@@ -832,7 +826,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify reason for conflict is 'wal_level_insufficient' in pg_replication_slots
 check_slots_conflict_reason('wal_level_', 'wal_level_insufficient');
-- 
2.43.5



  [application/octet-stream] v6-PG16-0001-Stabilize-035_standby_logical_decoding.pl.patch (7.5K, ../../OSCPR01MB14966BE900F2F29594D5E1FE7F5B52@OSCPR01MB14966.jpnprd01.prod.outlook.com/3-v6-PG16-0001-Stabilize-035_standby_logical_decoding.pl.patch)
  download | inline diff:
From a81a3b933bb1c2464f9a6442f2a4833d904cf628 Mon Sep 17 00:00:00 2001
From: Amit Kapila <[email protected]>
Date: Mon, 7 Apr 2025 17:58:23 +0530
Subject: [PATCH v6-PG16] Stabilize 035_standby_logical_decoding.pl.

Some tests try to invalidate logical slots on the standby server by
running VACUUM on the primary. The problem is that xl_running_xacts was
getting generated and replayed before the VACUUM command, leading to the
advancement of the active slot's catalog_xmin. Due to this, active slots
were not getting invalidated, leading to test failures.

We fix it by skipping the generation of xl_running_xacts for the required
tests with the help of injection points. As the required interface for
injection points was not present in back branches, we fixed the failing
tests in them by disallowing the slot to become active for the required
cases (where rows_removed conflict could be generated).
---
 .../t/035_standby_logical_decoding.pl         | 49 +++++++++++--------
 1 file changed, 29 insertions(+), 20 deletions(-)

diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 82ad7ce0c2b..360ed826094 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -205,9 +205,6 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 
 	change_hot_standby_feedback_and_wait_for_xmins($hsf, $invalidated);
 
-	$handle =
-	  make_slot_active($node_standby, $slot_prefix, 1, \$stdout, \$stderr);
-
 	# reset stat: easier to check for confl_active_logicalslot in pg_stat_database_conflicts
 	$node_standby->psql('testdb', q[select pg_stat_reset();]);
 }
@@ -215,7 +212,7 @@ sub reactive_slots_change_hfs_and_wait_for_xmins
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 sub check_for_invalidation
 {
-	my ($slot_prefix, $log_start, $test_name) = @_;
+	my ($slot_prefix, $log_start, $test_name, $checks_active_slot) = @_;
 
 	my $active_slot = $slot_prefix . 'activeslot';
 	my $inactive_slot = $slot_prefix . 'inactiveslot';
@@ -231,13 +228,17 @@ sub check_for_invalidation
 			$log_start),
 		"activeslot slot invalidation is logged $test_name");
 
-	# Verify that pg_stat_database_conflicts.confl_active_logicalslot has been updated
-	ok( $node_standby->poll_query_until(
-			'postgres',
-			"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
-			't'),
-		'confl_active_logicalslot updated'
-	) or die "Timed out waiting confl_active_logicalslot to be updated";
+	if ($checks_active_slot)
+	{
+		# Verify that pg_stat_database_conflicts.confl_active_logicalslot has
+		# been updated
+		ok( $node_standby->poll_query_until(
+				'postgres',
+				"select (confl_active_logicalslot = 1) from pg_stat_database_conflicts where datname = 'testdb'",
+				't'),
+			'confl_active_logicalslot updated'
+		) or die "Timed out waiting confl_active_logicalslot to be updated";
+	}
 }
 
 # Launch $sql query, wait for a new snapshot that has a newer horizon and
@@ -250,7 +251,11 @@ sub check_for_invalidation
 # seeing a xl_running_xacts that would advance an active replication slot's
 # catalog_xmin.  Advancing the active replication slot's catalog_xmin
 # would break some tests that expect the active slot to conflict with
-# the catalog xmin horizon.
+# the catalog xmin horizon.  Even with the above precaution, there is a risk
+# of xl_running_xacts record being logged and replayed before the VACUUM
+# command, leading to the test failure.  So, we ensured that replication slots
+# are not activated for tests that can invalidate slots due to 'rows_removed'
+# conflict reason.
 sub wait_until_vacuum_can_remove
 {
 	my ($vac_option, $sql, $to_vac) = @_;
@@ -532,7 +537,7 @@ $node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION tap_sub");
 $node_subscriber->stop;
 
 ##################################################
-# Recovery conflict: Invalidate conflicting slots, including in-use slots
+# Recovery conflict: Invalidate conflicting slots
 # Scenario 1: hot_standby_feedback off and vacuum FULL
 ##################################################
 
@@ -550,7 +555,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class');
+check_for_invalidation('vacuum_full_', 1, 'with vacuum FULL on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -620,7 +625,7 @@ ok(!-f "$standby_walfile",
 	"invalidated logical slots do not lead to retaining WAL");
 
 ##################################################
-# Recovery conflict: Invalidate conflicting slots, including in-use slots
+# Recovery conflict: Invalidate conflicting slots
 # Scenario 2: conflict due to row removal with hot_standby_feedback off.
 ##################################################
 
@@ -641,7 +646,7 @@ wait_until_vacuum_can_remove(
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class');
+check_for_invalidation('row_removal_', $logstart, 'with vacuum on pg_class', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -677,7 +682,7 @@ $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
 check_for_invalidation('shared_row_removal_', $logstart,
-	'with vacuum on pg_authid');
+	'with vacuum on pg_authid', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -701,6 +706,10 @@ $logstart = -s $node_standby->logfile;
 reactive_slots_change_hfs_and_wait_for_xmins('shared_row_removal_',
 	'no_conflict_', 0, 1);
 
+# As this scenario is not expected to produce any conflict, so activate the slot.
+# See comments atop wait_until_vacuum_can_remove().
+make_slot_active($node_standby, 'no_conflict_', 1, \$stdout, \$stderr);
+
 # This should not trigger a conflict
 wait_until_vacuum_can_remove(
 	'', 'CREATE TABLE conflict_test(x integer, y text);
@@ -738,7 +747,7 @@ change_hot_standby_feedback_and_wait_for_xmins(1, 0);
 $node_standby->restart;
 
 ##################################################
-# Recovery conflict: Invalidate conflicting slots, including in-use slots
+# Recovery conflict: Invalidate conflicting slots
 # Scenario 5: conflict due to on-access pruning.
 ##################################################
 
@@ -763,7 +772,7 @@ $node_primary->safe_psql('testdb', qq[UPDATE prun SET s = 'E';]);
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('pruning_', $logstart, 'with on-access pruning');
+check_for_invalidation('pruning_', $logstart, 'with on-access pruning', 0);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
@@ -807,7 +816,7 @@ $node_primary->restart;
 $node_primary->wait_for_replay_catchup($node_standby);
 
 # Check invalidation in the logfile and in pg_stat_database_conflicts
-check_for_invalidation('wal_level_', $logstart, 'due to wal_level');
+check_for_invalidation('wal_level_', $logstart, 'due to wal_level', 1);
 
 # Verify slots are reported as conflicting in pg_replication_slots
 check_slots_conflicting_status(1);
-- 
2.43.5



^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-08 05:13  Michael Paquier <[email protected]>
  parent: Hayato Kuroda (Fujitsu) <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Michael Paquier @ 2025-04-08 05:13 UTC (permalink / raw)
  To: Hayato Kuroda (Fujitsu) <[email protected]>; +Cc: 'Amit Kapila' <[email protected]>; Bertrand Drouvot <[email protected]>; [email protected] <[email protected]>

On Tue, Apr 08, 2025 at 02:00:35AM +0000, Hayato Kuroda (Fujitsu) wrote:
> Your patch looks good to me and it could pass on my env. PSA patches for PG16.
> Patch for PG17 is not changed, just renamed.

@@ -1287,6 +1288,17 @@ LogStandbySnapshot(void)
 
     Assert(XLogStandbyInfoActive());
 
+#ifdef USE_INJECTION_POINTS
+    if (IS_INJECTION_POINT_ATTACHED("skip-log-running-xacts"))
+    {
+        /*
+         * This record could move slot's xmin forward during decoding, leading
+         * to unpredictable results, so skip it when requested by the test.
+         */
+        return GetInsertRecPtr();
+    }
+#endif

I have unfortunately not been able to pay much attention to this
thread, but using an injection point as a trick to disable the
generation of these random standby snapshot records is an interesting
approach to stabilize the test, and it should make it faster as well.
Nice.
--
Michael


Attachments:

  [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
  download

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-08 06:19  Bertrand Drouvot <[email protected]>
  parent: Michael Paquier <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-08 06:19 UTC (permalink / raw)
  To: Michael Paquier <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi,

On Tue, Apr 08, 2025 at 02:13:20PM +0900, Michael Paquier wrote:
> On Tue, Apr 08, 2025 at 02:00:35AM +0000, Hayato Kuroda (Fujitsu) wrote:
> > Your patch looks good to me and it could pass on my env. PSA patches for PG16.
> > Patch for PG17 is not changed, just renamed.
> 
> @@ -1287,6 +1288,17 @@ LogStandbySnapshot(void)
>  
>      Assert(XLogStandbyInfoActive());
>  
> +#ifdef USE_INJECTION_POINTS
> +    if (IS_INJECTION_POINT_ATTACHED("skip-log-running-xacts"))
> +    {
> +        /*
> +         * This record could move slot's xmin forward during decoding, leading
> +         * to unpredictable results, so skip it when requested by the test.
> +         */
> +        return GetInsertRecPtr();
> +    }
> +#endif
> 
> I have unfortunately not been able to pay much attention to this
> thread, but using an injection point as a trick to disable the
> generation of these random standby snapshot records is an interesting
> approach to stabilize the test, and it should make it faster as well.
> Nice.

Yeah. That said I still think it could be useful to implement what has been
proposed in v1-0001 in [1] i.e:

- A new injection_points_wakeup_detach() function that is holding the spinlock
during the whole duration to ensure that no process can wait in between the
wakeup and the detach.

- injection_wait() should try to reuse an existing slot (if any) before trying
to use an empty one.

This is not needed here anymore (as we're using another injection point that the
one initially prooposed) but I'll open a dedicated thread for that for 19 when
the timing will be appropriate.

[1]: https://www.postgresql.org/message-id/Z6oQXc8LmiTLfwLA%40ip-10-97-1-34.eu-west-3.compute.internal

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-08 06:27  Michael Paquier <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Michael Paquier @ 2025-04-08 06:27 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

On Tue, Apr 08, 2025 at 06:19:02AM +0000, Bertrand Drouvot wrote:
> - A new injection_points_wakeup_detach() function that is holding the spinlock
> during the whole duration to ensure that no process can wait in between the
> wakeup and the detach.

That would not a correct spinlock use.  injection_points_detach() and
injection_points_wakeup_internal() do much more actions than what we
can internally do while holding a spinlock, including both
Postgres-specific calls as well as system calls.  strcmp() and
strlcpy() are still OK-ish, even as system calls, as they work
directly on the strings given in input.
--
Michael


Attachments:

  [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
  download

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-08 06:42  Bertrand Drouvot <[email protected]>
  parent: Michael Paquier <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-08 06:42 UTC (permalink / raw)
  To: Michael Paquier <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi,

On Tue, Apr 08, 2025 at 03:27:41PM +0900, Michael Paquier wrote:
> On Tue, Apr 08, 2025 at 06:19:02AM +0000, Bertrand Drouvot wrote:
> > - A new injection_points_wakeup_detach() function that is holding the spinlock
> > during the whole duration to ensure that no process can wait in between the
> > wakeup and the detach.
> 
> That would not a correct spinlock use.  injection_points_detach() and
> injection_points_wakeup_internal() do much more actions than what we
> can internally do while holding a spinlock,

Fully agree. Will need to find another way to prevent a process to wait between the
wakeup and the detach. I'll open a dedicated thread.

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-09 03:03  Michael Paquier <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Michael Paquier @ 2025-04-09 03:03 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

On Tue, Apr 08, 2025 at 06:42:53AM +0000, Bertrand Drouvot wrote:
> Fully agree. Will need to find another way to prevent a process to wait between the
> wakeup and the detach. I'll open a dedicated thread.

By the way, there is a small thing that's itching me a bit about the
change done in LogStandbySnapshot() for 105b2cb33617.  Could it be
useful for debugging to add a elog(DEBUG1) with the LSN returned by
GetInsertRecPtr() when taking the short path?  We don't have any
visibility when the shortcut path is taken, which seems annoying in
the long term if we use the injection point skip-log-running-xacts for
other tests, and I suspect that there will be some as the standby
snapshots can be really annoying in tests where we want a predictible
set of WAL records when wal_level is "replica" or "logical".
--
Michael


Attachments:

  [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
  download

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-09 05:54  Bertrand Drouvot <[email protected]>
  parent: Michael Paquier <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Bertrand Drouvot @ 2025-04-09 05:54 UTC (permalink / raw)
  To: Michael Paquier <[email protected]>; +Cc: Hayato Kuroda (Fujitsu) <[email protected]>; 'Amit Kapila' <[email protected]>; [email protected] <[email protected]>

Hi,

On Wed, Apr 09, 2025 at 12:03:06PM +0900, Michael Paquier wrote:
> On Tue, Apr 08, 2025 at 06:42:53AM +0000, Bertrand Drouvot wrote:
> > Fully agree. Will need to find another way to prevent a process to wait between the
> > wakeup and the detach. I'll open a dedicated thread.
> 
> By the way, there is a small thing that's itching me a bit about the
> change done in LogStandbySnapshot() for 105b2cb33617.  Could it be
> useful for debugging to add a elog(DEBUG1) with the LSN returned by
> GetInsertRecPtr() when taking the short path?  We don't have any
> visibility when the shortcut path is taken, which seems annoying in
> the long term if we use the injection point skip-log-running-xacts for
> other tests, and I suspect that there will be some as the standby
> snapshots can be really annoying in tests where we want a predictible
> set of WAL records when wal_level is "replica" or "logical".

Yeah, I also think that would be good to have some way to debug this. That's
why I did propose to generate a WAL record by making use of LogLogicalMessage()
instead of GetInsertRecPtr() (see [1]). We could generate "bypassing xl_running_xacts"
or such and bonus point that would advance the record ptr. 

Adding an elog(DEBUG1) could make sense too.

[1]: https://www.postgresql.org/message-id/Z%2Buko5kbw/ek/h0F%40ip-10-97-1-34.eu-west-3.compute.internal

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com





^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-09 06:37  Amit Kapila <[email protected]>
  parent: Bertrand Drouvot <[email protected]>
  0 siblings, 1 reply; 55+ messages in thread

From: Amit Kapila @ 2025-04-09 06:37 UTC (permalink / raw)
  To: Bertrand Drouvot <[email protected]>; +Cc: Michael Paquier <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; [email protected] <[email protected]>

On Wed, Apr 9, 2025 at 11:24 AM Bertrand Drouvot
<[email protected]> wrote:
>
> On Wed, Apr 09, 2025 at 12:03:06PM +0900, Michael Paquier wrote:
> > On Tue, Apr 08, 2025 at 06:42:53AM +0000, Bertrand Drouvot wrote:
> > > Fully agree. Will need to find another way to prevent a process to wait between the
> > > wakeup and the detach. I'll open a dedicated thread.
> >
> > By the way, there is a small thing that's itching me a bit about the
> > change done in LogStandbySnapshot() for 105b2cb33617.  Could it be
> > useful for debugging to add a elog(DEBUG1) with the LSN returned by
> > GetInsertRecPtr() when taking the short path?  We don't have any
> > visibility when the shortcut path is taken, which seems annoying in
> > the long term if we use the injection point skip-log-running-xacts for
> > other tests, and I suspect that there will be some as the standby
> > snapshots can be really annoying in tests where we want a predictible
> > set of WAL records when wal_level is "replica" or "logical".
>
> Yeah, I also think that would be good to have some way to debug this.
>

I can't think of a good reason to have this DEBUG1 as there is no
predictability of it getting generated even with tests using an
injection point. OTOH, I don't have any objections to it if you would
like to proceed with this.

-- 
With Regards,
Amit Kapila.






^ permalink  raw  reply  [nested|flat] 55+ messages in thread

* Re: Fix 035_standby_logical_decoding.pl race conditions
@ 2025-04-10 03:00  Michael Paquier <[email protected]>
  parent: Amit Kapila <[email protected]>
  0 siblings, 0 replies; 55+ messages in thread

From: Michael Paquier @ 2025-04-10 03:00 UTC (permalink / raw)
  To: Amit Kapila <[email protected]>; +Cc: Bertrand Drouvot <[email protected]>; Hayato Kuroda (Fujitsu) <[email protected]>; [email protected] <[email protected]>

On Wed, Apr 09, 2025 at 12:07:31PM +0530, Amit Kapila wrote:
> On Wed, Apr 9, 2025 at 11:24 AM Bertrand Drouvot
> <[email protected]> wrote:
> I can't think of a good reason to have this DEBUG1 as there is no
> predictability of it getting generated even with tests using an
> injection point. OTOH, I don't have any objections to it if you would
> like to proceed with this.

The non-predictability of the event is my reason, as it can be useful
to know this information when grabbing for specific patterns in the
logs between failed and successful run differences.  In short, I'd
like to think that we are OK here, still this information is free to
have and it could be useful if we still have problems.  A custom
message WAL record is overdoing in it a bit, IMO, an elog() with the
LSN returned should be enough.
--
Michael


Attachments:

  [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
  download

^ permalink  raw  reply  [nested|flat] 55+ messages in thread

end of thread, other threads:[~2025-04-10 03:00 UTC | newest]

Thread overview: 55+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-03-13 07:59 [PATCH v49 3/7] Make archiver process an auxiliary process Kyotaro Horiguchi <[email protected]>
2024-01-09 05:08 [PATCH v3] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v6] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v3] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v6] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v7] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v7] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v6] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v7] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v3] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v8] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v8] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-09 05:08 [PATCH v8] Fix 035_standby_logical_decoding.pl race condition bdrouvot <[email protected]>
2024-01-11 13:36 [PATCH v5] Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2024-01-11 13:36 [PATCH v4] Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2024-01-11 13:36 [PATCH v4] Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2024-01-11 13:36 [PATCH v5] Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2024-01-11 13:36 [PATCH v4] Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2024-01-11 13:36 [PATCH v5] Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-03-19 06:42 Re: Fix 035_standby_logical_decoding.pl race conditions Amit Kapila <[email protected]>
2025-03-19 10:26 ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-03-21 12:28   ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-03-21 16:18     ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-03-24 04:54       ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-03-25 03:44         ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-03-25 11:34       ` Re: Fix 035_standby_logical_decoding.pl race conditions Amit Kapila <[email protected]>
2025-03-26 07:47         ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-03-27 10:20           ` Re: Fix 035_standby_logical_decoding.pl race conditions Amit Kapila <[email protected]>
2025-03-28 09:02             ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-03-31 09:23               ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-01 01:22                 ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-01 08:32                   ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-01 11:52                     ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-01 15:15                     ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-02 07:16                   ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-02 08:36                     ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-02 11:14                       ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-02 12:13                       ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-02 15:00                         ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-03 05:34                           ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-03 05:58                             ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-08 02:00                               ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-08 05:13                                 ` Re: Fix 035_standby_logical_decoding.pl race conditions Michael Paquier <[email protected]>
2025-04-08 06:19                                   ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-08 06:27                                     ` Re: Fix 035_standby_logical_decoding.pl race conditions Michael Paquier <[email protected]>
2025-04-08 06:42                                       ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-09 03:03                                         ` Re: Fix 035_standby_logical_decoding.pl race conditions Michael Paquier <[email protected]>
2025-04-09 05:54                                           ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-09 06:37                                             ` Re: Fix 035_standby_logical_decoding.pl race conditions Amit Kapila <[email protected]>
2025-04-10 03:00                                               ` Re: Fix 035_standby_logical_decoding.pl race conditions Michael Paquier <[email protected]>
2025-04-03 06:59                             ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-07 06:15                               ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>
2025-04-07 09:10                                 ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-04-07 09:58                               ` Re: Fix 035_standby_logical_decoding.pl race conditions Bertrand Drouvot <[email protected]>
2025-03-26 10:56         ` RE: Fix 035_standby_logical_decoding.pl race conditions Hayato Kuroda (Fujitsu) <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox