Subject: Helper functions for wait_for_catchup() in Cluster.pm

From: "Drouvot, Bertrand" <[email-masked]>
Date: Tue, 17 Jan 2023 11:48:19 +0100

Hi hackers,

please find attached a patch proposal to define $SUBJECT.

The idea has been raised in [1], where we are adding more calls to wait_for_catchup() in 'replay' mode.

The current code already has 25 of those, so the proposed patch is defining a new wait_for_replay_catchup() function.

While at it, adding also:

- wait_for_write_catchup(): called 5 times
- wait_for_sent_catchup() and wait_for_flush_catchup() for consistency purpose (while there is
currently no occurrences of wait_for_catchup() in 'sent' or 'flush' mode.).

Looking forward to your feedback,

Regards,

-- 
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com

[1]: https://www.postgresql.org/message-id/20230106034036.2m4qnn7ep7b5ipet%40awork3.anarazel.de

diff --git a/src/bin/pg_rewind/t/007_standby_source.pl b/src/bin/pg_rewind/t/007_standby_source.pl
index 52644c2c0d..0a7e2d4521 100644
--- a/src/bin/pg_rewind/t/007_standby_source.pl
+++ b/src/bin/pg_rewind/t/007_standby_source.pl
@@ -75,8 +75,8 @@ $node_a->safe_psql('postgres',
 $node_a->safe_psql('postgres', 'CHECKPOINT');
 
 my $lsn = $node_a->lsn('write');
-$node_a->wait_for_catchup('node_b', 'write', $lsn);
-$node_b->wait_for_catchup('node_c', 'write', $lsn);
+$node_a->wait_for_write_catchup('node_b', $lsn);
+$node_b->wait_for_write_catchup('node_c', $lsn);
 
 # Promote C
 #
@@ -160,7 +160,7 @@ in A, after C was promoted
 $node_a->safe_psql('postgres',
 	"INSERT INTO tbl1 values ('in A, after rewind')");
 
-$node_b->wait_for_catchup('node_c', 'replay', $node_a->lsn('write'));
+$node_b->wait_for_replay_catchup('node_c', $node_a->lsn('write'));
 
 check_query(
 	'SELECT * FROM tbl1',
diff --git a/src/test/modules/brin/t/02_wal_consistency.pl b/src/test/modules/brin/t/02_wal_consistency.pl
index 5983ef208e..8d1dc68194 100644
--- a/src/test/modules/brin/t/02_wal_consistency.pl
+++ b/src/test/modules/brin/t/02_wal_consistency.pl
@@ -70,6 +70,6 @@ my ($ret, $out, $err) = $whiskey->psql(
 	});
 cmp_ok($out, '>=', 1);
 
-$whiskey->wait_for_catchup($charlie, 'replay', $whiskey->lsn('insert'));
+$whiskey->wait_for_replay_catchup($charlie, $whiskey->lsn('insert'));
 
 done_testing();
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 04921ca3a3..3e3aeea0c6 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2711,6 +2711,70 @@ sub wait_for_catchup
 
 =pod
 
+=item $node->wait_for_flush_catchup(standby_name, target_lsn)
+
+Helper function for wait_for_catchup when waiting for the flush_lsn column
+to catchup.
+
+=cut
+
+sub wait_for_flush_catchup
+{
+	my ($self, $standby_name, $target_lsn) = @_;
+
+	$self->wait_for_catchup($standby_name, 'flush', $target_lsn);
+}
+
+=pod
+
+=item $node->wait_for_replay_catchup(standby_name, target_lsn)
+
+Helper function for wait_for_catchup when waiting for the replay_lsn column
+to catchup.
+
+=cut
+
+sub wait_for_replay_catchup
+{
+	my ($self, $standby_name, $target_lsn) = @_;
+
+	$self->wait_for_catchup($standby_name, 'replay', $target_lsn);
+}
+
+=pod
+
+=item $node->wait_for_sent_catchup(standby_name, target_lsn)
+
+Helper function for wait_for_catchup when waiting for the sent_lsn column
+to catchup.
+
+=cut
+
+sub wait_for_sent_catchup
+{
+	my ($self, $standby_name, $target_lsn) = @_;
+
+	$self->wait_for_catchup($standby_name, 'sent', $target_lsn);
+}
+
+=pod
+
+=item $node->wait_for_write_catchup(standby_name, target_lsn)
+
+Helper function for wait_for_catchup when waiting for the write_lsn column
+to catchup.
+
+=cut
+
+sub wait_for_write_catchup
+{
+	my ($self, $standby_name, $target_lsn) = @_;
+
+	$self->wait_for_catchup($standby_name, 'write', $target_lsn);
+}
+
+=pod
+
 =item $node->wait_for_slot_catchup(slot_name, mode, target_lsn)
 
 Wait for the named replication slot to equal or pass the supplied target_lsn.
diff --git a/src/test/recovery/t/001_stream_rep.pl b/src/test/recovery/t/001_stream_rep.pl
index 23a90dd85b..c90f9c8383 100644
--- a/src/test/recovery/t/001_stream_rep.pl
+++ b/src/test/recovery/t/001_stream_rep.pl
@@ -48,8 +48,8 @@ $node_primary->safe_psql('postgres',
 
 # Wait for standbys to catch up
 my $primary_lsn = $node_primary->lsn('write');
-$node_primary->wait_for_catchup($node_standby_1, 'replay', $primary_lsn);
-$node_standby_1->wait_for_catchup($node_standby_2, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby_1, $primary_lsn);
+$node_standby_1->wait_for_replay_catchup($node_standby_2, $primary_lsn);
 
 my $result =
   $node_standby_1->safe_psql('postgres', "SELECT count(*) FROM tab_int");
@@ -67,8 +67,8 @@ $node_primary->safe_psql('postgres',
 
 # Wait for standbys to catch up
 $primary_lsn = $node_primary->lsn('write');
-$node_primary->wait_for_catchup($node_standby_1, 'replay', $primary_lsn);
-$node_standby_1->wait_for_catchup($node_standby_2, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby_1, $primary_lsn);
+$node_standby_1->wait_for_replay_catchup($node_standby_2, $primary_lsn);
 
 $result = $node_standby_1->safe_psql('postgres', "SELECT * FROM seq1");
 print "standby 1: $result\n";
@@ -373,9 +373,8 @@ sub replay_check
 		'INSERT INTO replayed(val) SELECT coalesce(max(val),0) + 1 AS newval FROM replayed RETURNING val'
 	);
 	my $primary_lsn = $node_primary->lsn('write');
-	$node_primary->wait_for_catchup($node_standby_1, 'replay', $primary_lsn);
-	$node_standby_1->wait_for_catchup($node_standby_2, 'replay',
-		$primary_lsn);
+	$node_primary->wait_for_replay_catchup($node_standby_1, $primary_lsn);
+	$node_standby_1->wait_for_replay_catchup($node_standby_2, $primary_lsn);
 
 	$node_standby_1->safe_psql('postgres',
 		qq[SELECT 1 FROM replayed WHERE val = $newval])
diff --git a/src/test/recovery/t/010_logical_decoding_timelines.pl b/src/test/recovery/t/010_logical_decoding_timelines.pl
index eb1a3b6ef8..2cd8b1a246 100644
--- a/src/test/recovery/t/010_logical_decoding_timelines.pl
+++ b/src/test/recovery/t/010_logical_decoding_timelines.pl
@@ -135,7 +135,7 @@ cmp_ok(
 	'xmin on physical slot must not be lower than catalog_xmin');
 
 $node_primary->safe_psql('postgres', 'CHECKPOINT');
-$node_primary->wait_for_catchup($node_replica, 'write');
+$node_primary->wait_for_write_catchup($node_replica);
 
 # Boom, crash
 $node_primary->stop('immediate');
diff --git a/src/test/recovery/t/027_stream_regress.pl b/src/test/recovery/t/027_stream_regress.pl
index 69d6ddf281..3e75cb593a 100644
--- a/src/test/recovery/t/027_stream_regress.pl
+++ b/src/test/recovery/t/027_stream_regress.pl
@@ -86,8 +86,7 @@ $node_primary->psql('regression',
 	"select setval(seqrelid, nextval(seqrelid)) from pg_sequence");
 
 # Wait for standby to catch up
-$node_primary->wait_for_catchup($node_standby_1, 'replay',
-	$node_primary->lsn('insert'));
+$node_primary->wait_for_replay_catchup($node_standby_1, $node_primary->lsn('insert'));
 
 # Perform a logical dump of primary and standby, and check that they match
 command_ok(
diff --git a/src/test/recovery/t/030_stats_cleanup_replica.pl b/src/test/recovery/t/030_stats_cleanup_replica.pl
index f1121e4b12..76abbae807 100644
--- a/src/test/recovery/t/030_stats_cleanup_replica.pl
+++ b/src/test/recovery/t/030_stats_cleanup_replica.pl
@@ -39,7 +39,7 @@ drop_function_by_oid('postgres', $funcoid);
 
 $sect = 'post drop';
 my $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 test_standby_func_tab_stats_status('postgres',
 	$dboid, $tableoid, $funcoid, 'f');
 
@@ -50,7 +50,7 @@ $sect = "schema creation";
 
 $node_primary->safe_psql('postgres', "CREATE SCHEMA drop_schema_test1");
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 ($dboid, $tableoid, $funcoid) =
   populate_standby_stats('postgres', 'drop_schema_test1');
@@ -62,7 +62,7 @@ $node_primary->safe_psql('postgres', "DROP SCHEMA drop_schema_test1 CASCADE");
 $sect = "post schema drop";
 
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 # verify table and function stats removed from standby
 test_standby_func_tab_stats_status('postgres',
@@ -75,7 +75,7 @@ $sect = "createdb";
 
 $node_primary->safe_psql('postgres', "CREATE DATABASE test");
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 ($dboid, $tableoid, $funcoid) = populate_standby_stats('test', 'public');
 
@@ -86,7 +86,7 @@ test_standby_db_stats_status('test', $dboid, 't');
 $node_primary->safe_psql('postgres', "DROP DATABASE test");
 $sect        = "post dropdb";
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 # Test that the stats were cleaned up on standby
 # Note that this connects to 'postgres' but provides the dboid of dropped db
@@ -138,7 +138,7 @@ sub populate_standby_stats
 		"CREATE FUNCTION $schema.drop_func_test1() RETURNS VOID AS 'select 2;' LANGUAGE SQL IMMUTABLE"
 	);
 	my $primary_lsn = $node_primary->lsn('flush');
-	$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+	$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 	# collect object oids
 	my $dboid = $node_standby->safe_psql($connect_db,
diff --git a/src/test/recovery/t/031_recovery_conflict.pl b/src/test/recovery/t/031_recovery_conflict.pl
index 875afb8e3c..900e862d4f 100644
--- a/src/test/recovery/t/031_recovery_conflict.pl
+++ b/src/test/recovery/t/031_recovery_conflict.pl
@@ -64,7 +64,7 @@ INSERT INTO $table1 SELECT i % 3, 0 FROM generate_series(1,20) i;
 CREATE TABLE ${table2}(a int, b int);
 ]);
 my $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);

 # a longrunning psql that we can use to trigger conflicts
@@ -98,7 +98,7 @@ $node_primary->safe_psql(
 	]);
 
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 my $cursor1 = "test_recovery_conflict_cursor";
 
@@ -125,7 +125,7 @@ $node_primary->safe_psql($test_db, qq[VACUUM $table1;]);
 # encountering the recovery conflict which causes the disconnect and checking
 # the logfile for the terminated connection.
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 check_conflict_log("User was holding shared buffer pin for too long");
 reconnect_and_clear();
@@ -139,7 +139,7 @@ $expected_conflicts++;
 $node_primary->safe_psql($test_db,
 	qq[INSERT INTO $table1 SELECT i, 0 FROM generate_series(1,20) i]);
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 # DECLARE and FETCH from cursor on the standby
 $psql_standby{stdin} .= qq[
@@ -161,7 +161,7 @@ $node_primary->safe_psql($test_db, qq[VACUUM $table1;]);
 
 # Wait for attempted replay of PRUNE records
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 check_conflict_log(
 	"User query might have needed to see row versions that must be removed");
@@ -185,7 +185,7 @@ ok(pump_until_standby(qr/^1$/m), "$sect: conflicting lock acquired");
 $node_primary->safe_psql($test_db, qq[DROP TABLE $table1;]);
 
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 check_conflict_log("User was holding a relation lock for too long");
 reconnect_and_clear();
@@ -214,7 +214,7 @@ ok(pump_until_standby(qr/^6000$/m),
 $node_primary->safe_psql($test_db, qq[DROP TABLESPACE $tablespace1;]);
 
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 check_conflict_log(
 	"User was or might have been using tablespace that must be dropped");
@@ -256,7 +256,7 @@ SELECT txid_current();
 ]);
 
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 $psql_standby{stdin} .= qq[
     BEGIN;
@@ -283,7 +283,7 @@ SELECT 'waiting' FROM pg_locks WHERE locktype = 'relation' AND NOT granted;
 # psql is waiting on lock
 $node_primary->safe_psql($test_db, qq[VACUUM $table1;]);
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 check_conflict_log("User transaction caused buffer deadlock with recovery.");
 reconnect_and_clear();
@@ -312,7 +312,7 @@ $sect = "database conflict";
 $node_primary->safe_psql('postgres', qq[DROP DATABASE $test_db;]);
 
 $primary_lsn = $node_primary->lsn('flush');
-$node_primary->wait_for_catchup($node_standby, 'replay', $primary_lsn);
+$node_primary->wait_for_replay_catchup($node_standby, $primary_lsn);
 
 check_conflict_log("User was connected to a database that must be dropped");
 
diff --git a/src/test/recovery/t/033_replay_tsp_drops.pl b/src/test/recovery/t/033_replay_tsp_drops.pl
index 896b282bd4..58f142f459 100644
--- a/src/test/recovery/t/033_replay_tsp_drops.pl
+++ b/src/test/recovery/t/033_replay_tsp_drops.pl
@@ -42,8 +42,7 @@ sub test_tablespace
 	$node_standby->start;
 
 	# Make sure the connection is made
-	$node_primary->wait_for_catchup($node_standby, 'write',
-		$node_primary->lsn('write'));
+	$node_primary->wait_for_write_catchup($node_standby, $node_primary->lsn('write'));
 
 	# Do immediate shutdown just after a sequence of CREATE DATABASE / DROP
 	# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
@@ -65,8 +64,7 @@ sub test_tablespace
 	$query =~ s/<STRATEGY>/$strategy/g;
 
 	$node_primary->safe_psql('postgres', $query);
-	$node_primary->wait_for_catchup($node_standby, 'write',
-		$node_primary->lsn('write'));
+	$node_primary->wait_for_write_catchup($node_standby, $node_primary->lsn('write'));
 
 	# show "create missing directory" log message
 	$node_standby->safe_psql('postgres',