Subject: Re: Supply restore_command to pg_rewind via CLI argument

From: Andres Freund <[email-masked]>
Date: Mon, 21 Mar 2022 17:32:06 -0700

Hi,

On 2021-11-05 15:10:29 +0500, Andrey Borodin wrote:
> > 4 нояб. 2021 г., в 17:55, Daniel Gustafsson <[email-masked]> написал(а):
> > 
> > The patch no longer applies, can you submit a rebased version please?
> 
> Thanks, Daniel! PFA rebase.

Doesn't apply once more: http://cfbot.cputube.org/patch_37_3213.log

Greetings,

Andres Freund

---

From: Alexey Kondratov <[email-masked]>
Date: Tue, 22 Mar 2022 12:23:35 +0300

Hi,

> Doesn't apply once more: http://cfbot.cputube.org/patch_37_3213.log

Thanks for the reminder, a rebased version is attached.

Regards
-- 
Alexey Kondratov
From df56b5c7b882e781fdc0b92e7a83331f0baab094 Mon Sep 17 00:00:00 2001
From: Alexey Kondratov <[email-masked]>
Date: Tue, 29 Jun 2021 17:17:47 +0300
Subject: [PATCH v4] Allow providing restore_command as a command line option
 to pg_rewind

This could be useful when postgres is usually run with
-c config_file=..., so the actual configuration and restore_command
is not inside $PGDATA/postgresql.conf.
---
 doc/src/sgml/ref/pg_rewind.sgml          | 19 +++++
 src/bin/pg_rewind/pg_rewind.c            | 45 ++++++-----
 src/bin/pg_rewind/t/001_basic.pl         |  1 +
 src/bin/pg_rewind/t/RewindTest.pm        | 95 ++++++++++++++----------
 src/test/perl/PostgreSQL/Test/Cluster.pm |  5 +-
 5 files changed, 106 insertions(+), 59 deletions(-)

diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml
index 33e6bb64ad..af75f35867 100644
--- a/doc/src/sgml/ref/pg_rewind.sgml
+++ b/doc/src/sgml/ref/pg_rewind.sgml
@@ -241,6 +241,25 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-C <replaceable class="parameter">restore_command</replaceable></option></term>
+      <term><option>--target-restore-command=<replaceable class="parameter">restore_command</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the <varname>restore_command</varname> to use for retrieving
+        WAL files from the WAL archive if these files are no longer available
+        in the <filename>pg_wal</filename> directory of the target cluster.
+       </para>
+       <para>
+        If <varname>restore_command</varname> is already set in
+        <filename>postgresql.conf</filename>, you can provide the
+        <option>--restore-target-wal</option> option instead. If both options
+        are provided, then <option>--target-restore-command</option>
+        will be used.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>--debug</option></term>
       <listitem>
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index b39b5c1aac..9aca041425 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -85,21 +85,22 @@ usage(const char *progname)
 	printf(_("%s resynchronizes a PostgreSQL cluster with another copy of the cluster.\n\n"), progname);
 	printf(_("Usage:\n  %s [OPTION]...\n\n"), progname);
 	printf(_("Options:\n"));
-	printf(_("  -c, --restore-target-wal       use restore_command in target configuration to\n"
-			 "                                 retrieve WAL files from archives\n"));
-	printf(_("  -D, --target-pgdata=DIRECTORY  existing data directory to modify\n"));
-	printf(_("      --source-pgdata=DIRECTORY  source data directory to synchronize with\n"));
-	printf(_("      --source-server=CONNSTR    source server to synchronize with\n"));
-	printf(_("  -n, --dry-run                  stop before modifying anything\n"));
-	printf(_("  -N, --no-sync                  do not wait for changes to be written\n"
-			 "                                 safely to disk\n"));
-	printf(_("  -P, --progress                 write progress messages\n"));
-	printf(_("  -R, --write-recovery-conf      write configuration for replication\n"
-			 "                                 (requires --source-server)\n"));
-	printf(_("      --debug                    write a lot of debug messages\n"));
-	printf(_("      --no-ensure-shutdown       do not automatically fix unclean shutdown\n"));
-	printf(_("  -V, --version                  output version information, then exit\n"));
-	printf(_("  -?, --help                     show this help, then exit\n"));
+	printf(_("  -c, --restore-target-wal              use restore_command in target configuration to\n"
+			 "                                        retrieve WAL files from archives\n"));
+	printf(_("  -C, --target-restore-command=COMMAND  target WAL restore_command\n"));
+	printf(_("  -D, --target-pgdata=DIRECTORY         existing data directory to modify\n"));
+	printf(_("      --source-pgdata=DIRECTORY         source data directory to synchronize with\n"));
+	printf(_("      --source-server=CONNSTR           source server to synchronize with\n"));
+	printf(_("  -n, --dry-run                         stop before modifying anything\n"));
+	printf(_("  -N, --no-sync                         do not wait for changes to be written\n"
+			 "                                        safely to disk\n"));
+	printf(_("  -P, --progress                        write progress messages\n"));
+	printf(_("  -R, --write-recovery-conf             write configuration for replication\n"
+			 "                                        (requires --source-server)\n"));
+	printf(_("      --debug                           write a lot of debug messages\n"));
+	printf(_("      --no-ensure-shutdown              do not automatically fix unclean shutdown\n"));
+	printf(_("  -V, --version                         output version information, then exit\n"));
+	printf(_("  -?, --help                            show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
 	printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
 }
@@ -117,6 +118,7 @@ main(int argc, char **argv)
 		{"no-ensure-shutdown", no_argument, NULL, 4},
 		{"version", no_argument, NULL, 'V'},
 		{"restore-target-wal", no_argument, NULL, 'c'},
+		{"target-restore-command", required_argument, NULL, 'C'},
 		{"dry-run", no_argument, NULL, 'n'},
 		{"no-sync", no_argument, NULL, 'N'},
 		{"progress", no_argument, NULL, 'P'},
@@ -157,7 +159,7 @@ main(int argc, char **argv)
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "cD:nNPR", long_options, &option_index)) != -1)
+	while ((c = getopt_long(argc, argv, "cC:D:nNPR", long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
@@ -169,6 +171,11 @@ main(int argc, char **argv)
 				restore_wal = true;
 				break;
 
+			case 'C':
+				restore_wal = true;
+				restore_command = pg_strdup(optarg);
+				break;
+
 			case 'P':
 				showprogress = true;
 				break;
@@ -1020,7 +1027,11 @@ getRestoreCommand(const char *argv0)
 				cmd_output[MAXPGPATH];
 	PQExpBuffer postgres_cmd;
 
-	if (!restore_wal)
+	/*
+	 * Take restore_command from the postgresql.conf only if it is not already
+	 * provided as a command line option.
+	 */
+	if (!restore_wal || restore_command != NULL)
 		return;
 
 	/* find postgres executable */
diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl
index db9201f38e..3c395ece12 100644
--- a/src/bin/pg_rewind/t/001_basic.pl
+++ b/src/bin/pg_rewind/t/001_basic.pl
@@ -190,5 +190,6 @@ in primary, before promotion
 run_test('local');
 run_test('remote');
 run_test('archive');
+run_test('archive_cli');
 
 done_testing();
diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm
index 5651602858..335f1a72ad 100644
--- a/src/bin/pg_rewind/t/RewindTest.pm
+++ b/src/bin/pg_rewind/t/RewindTest.pm
@@ -208,6 +208,58 @@ sub promote_standby
 	return;
 }
 
+sub run_pg_rewind_archive
+{
+	my $test_mode = shift;
+	my $restore_command;
+	my $cli_option = '--restore-target-wal';
+
+	# Remove the existing archive directory and move all WAL
+	# segments from the old primary to the archives.  These
+	# will be used by pg_rewind.
+	rmtree($node_primary->archive_dir);
+	PostgreSQL::Test::RecursiveCopy::copypath($node_primary->data_dir . '/pg_wal',
+		$node_primary->archive_dir);
+
+	# Fast way to remove entire directory content
+	rmtree($node_primary->data_dir . '/pg_wal');
+	mkdir($node_primary->data_dir . '/pg_wal');
+
+	# Make sure that directories have the right umask as this is
+	# required by a follow-up check on permissions, and better
+	# safe than sorry.
+	chmod(0700, $node_primary->archive_dir);
+	chmod(0700, $node_primary->data_dir . '/pg_wal');
+
+	# Add appropriate restore_command to the target cluster
+	$restore_command = $node_primary->enable_restoring($node_primary, 0);
+
+	# In archive_cli mode pass restore_command as a command line option
+	# instead of taking it from postgresql.conf
+	if ($test_mode eq 'archive_cli') {
+		$cli_option = "--target-restore-command=$restore_command";
+	}
+
+	# Stop the new primary and be ready to perform the rewind.
+	$node_standby->stop;
+
+	# Note the use of --no-ensure-shutdown here.  WAL files are
+	# gone in this mode and the primary has been stopped
+	# gracefully already.
+	command_ok(
+		[
+			'pg_rewind',
+			'--debug',
+			'--source-pgdata=' . $node_standby->data_dir,
+			'--target-pgdata=' . $node_primary->data_dir,
+			'--no-sync',
+			'--no-ensure-shutdown',
+			$cli_option
+		],
+		"pg_rewind $test_mode");
+
+}
+
 sub run_pg_rewind
 {
 	my $test_mode       = shift;
@@ -219,7 +271,7 @@ sub run_pg_rewind
 	# Append the rewind-specific role to the connection string.
 	$standby_connstr = "$standby_connstr user=rewind_user";
 
-	if ($test_mode eq 'archive')
+	if ($test_mode eq 'archive' or $test_mode eq 'archive_cli')
 	{
 		# pg_rewind is tested with --restore-target-wal by moving all
 		# WAL files to a secondary location.  Note that this leads to
@@ -292,50 +344,13 @@ sub run_pg_rewind
 		$node_standby->safe_psql('postgres',
 			"ALTER ROLE rewind_user WITH REPLICATION;");
 	}
-	elsif ($test_mode eq "archive")
+	elsif ($test_mode eq "archive" or $test_mode eq "archive_cli")
 	{
-
 		# Do rewind using a local pgdata as source and specified
 		# directory with target WAL archive.  The old primary has
 		# to be stopped at this point.
 
-		# Remove the existing archive directory and move all WAL
-		# segments from the old primary to the archives.  These
-		# will be used by pg_rewind.
-		rmtree($node_primary->archive_dir);
-		PostgreSQL::Test::RecursiveCopy::copypath($node_primary->data_dir . "/pg_wal",
-			$node_primary->archive_dir);
-
-		# Fast way to remove entire directory content
-		rmtree($node_primary->data_dir . "/pg_wal");
-		mkdir($node_primary->data_dir . "/pg_wal");
-
-		# Make sure that directories have the right umask as this is
-		# required by a follow-up check on permissions, and better
-		# safe than sorry.
-		chmod(0700, $node_primary->archive_dir);
-		chmod(0700, $node_primary->data_dir . "/pg_wal");
-
-		# Add appropriate restore_command to the target cluster
-		$node_primary->enable_restoring($node_primary, 0);
-
-		# Stop the new primary and be ready to perform the rewind.
-		$node_standby->stop;
-
-		# Note the use of --no-ensure-shutdown here.  WAL files are
-		# gone in this mode and the primary has been stopped
-		# gracefully already.
-		command_ok(
-			[
-				'pg_rewind',
-				"--debug",
-				"--source-pgdata=$standby_pgdata",
-				"--target-pgdata=$primary_pgdata",
-				"--no-sync",
-				"--no-ensure-shutdown",
-				"--restore-target-wal"
-			],
-			'pg_rewind archive');
+		run_pg_rewind_archive($test_mode);
 	}
 	else
 	{
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index e7b9161137..51e289aaa0 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -1071,7 +1071,8 @@ primary_conninfo='$root_connstr'
 	return;
 }
 
-# Internal routine to enable archive recovery command on a standby node
+# Internal routine to enable archive recovery command on a standby node.
+# Returns generated restore_command.
 sub enable_restoring
 {
 	my ($self, $root_node, $standby) = @_;
@@ -1104,7 +1105,7 @@ restore_command = '$copy_command'
 	{
 		$self->set_recovery_mode();
 	}
-	return;
+	return $copy_command;
 }
 
 =pod

base-commit: f5576a21b0778f275d7418f6f7a44d9400ee90aa
-- 
2.32.0