Subject: [PATCH v4] pg_rewind: options to use restore_command from command line or cluster config

From: Alexey Kondratov <[email-masked]>
Date: Tue, 19 Feb 2019 19:14:53 +0300

---
 doc/src/sgml/ref/pg_rewind.sgml       |  30 ++++-
 src/bin/pg_rewind/parsexlog.c         | 161 +++++++++++++++++++++++++-
 src/bin/pg_rewind/pg_rewind.c         |  98 +++++++++++++++-
 src/bin/pg_rewind/pg_rewind.h         |   7 +-
 src/bin/pg_rewind/t/001_basic.pl      |   4 +-
 src/bin/pg_rewind/t/002_databases.pl  |   4 +-
 src/bin/pg_rewind/t/003_extrafiles.pl |   4 +-
 src/bin/pg_rewind/t/RewindTest.pm     |  84 +++++++++++++-
 8 files changed, 372 insertions(+), 20 deletions(-)

diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml
index 53a64ee29e..0c2441afa7 100644
--- a/doc/src/sgml/ref/pg_rewind.sgml
+++ b/doc/src/sgml/ref/pg_rewind.sgml
@@ -67,8 +67,10 @@ PostgreSQL documentation
    ancestor. In the typical failover scenario where the target cluster was
    shut down soon after the divergence, this is not a problem, but if the
    target cluster ran for a long time after the divergence, the old WAL
-   files might no longer be present. In that case, they can be manually
-   copied from the WAL archive to the <filename>pg_wal</filename> directory, or
+   files might no longer be present. In that case, they can be automatically
+   copied by <application>pg_rewind</application> from the WAL archive to the 
+   <filename>pg_wal</filename> directory if either <literal>-r</literal> or
+   <literal>-R</literal> option is specified, or
    fetched on startup by configuring <xref linkend="guc-primary-conninfo"/> or
    <xref linkend="guc-restore-command"/>.  The use of
    <application>pg_rewind</application> is not limited to failover, e.g.  a standby
@@ -200,6 +202,30 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-r</option></term>
+      <term><option>--use-postgresql-conf</option></term>
+      <listitem>
+       <para>
+        Use restore_command in the <filename>postgresql.conf</filename> to
+        retreive missing in the target <filename>pg_wal</filename> directory
+        WAL files from the WAL archive.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>-R <replaceable class="parameter">restore_command</replaceable></option></term>
+      <term><option>--restore-command=<replaceable class="parameter">restore_command</replaceable></option></term>
+      <listitem>
+       <para>
+        Specifies the restore_command to use for retrieval of the missing
+        in the target <filename>pg_wal</filename> directory WAL files from
+        the WAL archive.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>--debug</option></term>
       <listitem>
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index e19c265cbb..5978ec9b99 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -12,6 +12,7 @@
 #include "postgres_fe.h"
 
 #include <unistd.h>
+#include <sys/stat.h>
 
 #include "pg_rewind.h"
 #include "filemap.h"
@@ -45,6 +46,7 @@ static char xlogfpath[MAXPGPATH];
 typedef struct XLogPageReadPrivate
 {
 	const char *datadir;
+	const char *restoreCommand;
 	int			tliIndex;
 } XLogPageReadPrivate;
 
@@ -53,6 +55,9 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
 				   int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
 				   TimeLineID *pageTLI);
 
+static int RestoreArchivedWAL(const char *path, const char *xlogfname,
+				   off_t expectedSize, const char *restoreCommand);
+
 /*
  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
@@ -60,7 +65,7 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
  */
 void
 extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
-			   XLogRecPtr endpoint)
+			   XLogRecPtr endpoint, const char *restore_command)
 {
 	XLogRecord *record;
 	XLogReaderState *xlogreader;
@@ -69,6 +74,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	private.datadir = datadir;
 	private.tliIndex = tliIndex;
+	private.restoreCommand = restore_command;
 	xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
 									&private);
 	if (xlogreader == NULL)
@@ -156,7 +162,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 void
 findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 				   XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
-				   XLogRecPtr *lastchkptredo)
+				   XLogRecPtr *lastchkptredo, const char *restoreCommand)
 {
 	/* Walk backwards, starting from the given record */
 	XLogRecord *record;
@@ -181,6 +187,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 
 	private.datadir = datadir;
 	private.tliIndex = tliIndex;
+	private.restoreCommand = restoreCommand;
 	xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
 									&private);
 	if (xlogreader == NULL)
@@ -291,9 +298,30 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
 
 		if (xlogreadfd < 0)
 		{
-			printf(_("could not open file \"%s\": %s\n"), xlogfpath,
-				   strerror(errno));
-			return -1;
+			/*
+			 * If we have no restore_command to execute, then exit.
+			 */
+			if (private->restoreCommand == NULL)
+			{
+				printf(_("could not open file \"%s\": %s\n"), xlogfpath,
+					   strerror(errno));
+				return -1;
+			}
+
+			/*
+			 * Since we have restore_command to execute, then try to retreive
+			 * missing WAL file from the archive.
+			 */
+			xlogreadfd = RestoreArchivedWAL(private->datadir,
+											xlogfname,
+											WalSegSz,
+											private->restoreCommand);
+
+			if (xlogreadfd < 0)
+				return -1;
+			else
+				pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n",
+					   xlogfpath);
 		}
 	}
 
@@ -409,3 +437,126 @@ extractPageInfo(XLogReaderState *record)
 		process_block_change(forknum, rnode, blkno);
 	}
 }
+
+/*
+ * Attempt to retrieve the specified file from off-line archival storage.
+ * If successful return a file descriptor of restored WAL file, else
+ * return -1.
+ *
+ * For fixed-size files, the caller may pass the expected size as an
+ * additional crosscheck on successful recovery. If the file size is not
+ * known, set expectedSize = 0.
+ *
+ * This is a simplified and adapted to frontend version of
+ * RestoreArchivedFile function from transam/xlogarchive.c
+ */
+static int
+RestoreArchivedWAL(const char *path, const char *xlogfname,
+				   off_t expectedSize, const char *restoreCommand)
+{
+	char		xlogpath[MAXPGPATH],
+				xlogRestoreCmd[MAXPGPATH],
+			   *dp,
+			   *endp;
+	const char *sp;
+	int			rc,
+				xlogfd;
+	struct stat stat_buf;
+
+	snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);
+
+	/*
+	 * Construct the command to be executed.
+	 */
+	dp = xlogRestoreCmd;
+	endp = xlogRestoreCmd + MAXPGPATH - 1;
+	*endp = '\0';
+
+	for (sp = restoreCommand; *sp; sp++)
+	{
+		if (*sp == '%')
+		{
+			switch (sp[1])
+			{
+				case 'p':
+					/* %p: relative path of target file */
+					sp++;
+					StrNCpy(dp, xlogpath, endp - dp);
+					make_native_path(dp);
+					dp += strlen(dp);
+					break;
+				case 'f':
+					/* %f: filename of desired file */
+					sp++;
+					StrNCpy(dp, xlogfname, endp - dp);
+					dp += strlen(dp);
+					break;
+				case 'r':
+					/* %r: filename of last restartpoint */
+					pg_fatal("restore_command with %%r cannot be used during rewind process.\n");
+					break;
+				case '%':
+					/* convert %% to a single % */
+					sp++;
+					if (dp < endp)
+						*dp++ = *sp;
+					break;
+				default:
+					/* otherwise treat the % as not special */
+					if (dp < endp)
+						*dp++ = *sp;
+					break;
+			}
+		}
+		else
+		{
+			if (dp < endp)
+				*dp++ = *sp;
+		}
+	}
+	*dp = '\0';
+
+	/*
+	 * Execute restore_command, which should copy
+	 * the missing WAL file from archival storage.
+	 */
+	rc = system(xlogRestoreCmd);
+
+	if (rc == 0)
+	{
+		/*
+		 * Command apparently succeeded, but let's make sure the file is
+		 * really there now and has the correct size.
+		 */
+		if (stat(xlogpath, &stat_buf) == 0)
+		{
+			if (expectedSize > 0 && stat_buf.st_size != expectedSize)
+			{
+				printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"),
+						xlogfname, (unsigned long) stat_buf.st_size,
+						(unsigned long) expectedSize, strerror(errno));
+			}
+			else
+			{
+				xlogfd = open(xlogpath, O_RDONLY | PG_BINARY, 0);
+
+				if (xlogfd < 0)
+					printf(_("could not open restored from archive file \"%s\": %s\n"),
+							xlogpath, strerror(errno));
+				else
+					return xlogfd;
+			}
+		}
+		else
+		{
+			/* Stat failed */
+			printf(_("could not stat file \"%s\": %s"),
+					xlogpath, strerror(errno));
+		}
+	}
+
+	printf(_("could not restore file \"%s\" from archive: %s\n"),
+			xlogfname, strerror(errno));
+
+	return -1;
+}
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index 7ccde5c87f..9da578ec42 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -30,6 +30,8 @@
 #include "getopt_long.h"
 #include "storage/bufpage.h"
 
+#define MAX_RESTORE_COMMAND 1024
+
 static void usage(const char *progname);
 
 static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli,
@@ -52,11 +54,13 @@ int			WalSegSz;
 char	   *datadir_target = NULL;
 char	   *datadir_source = NULL;
 char	   *connstr_source = NULL;
+char	   *restore_command = NULL;
 
 bool		debug = false;
 bool		showprogress = false;
 bool		dry_run = false;
 bool		do_sync = true;
+bool		restore_wals = false;
 
 /* Target history */
 TimeLineHistoryEntry *targetHistory;
@@ -75,6 +79,9 @@ usage(const char *progname)
 	printf(_("  -N, --no-sync                  do not wait for changes to be written\n"));
 	printf(_("                                 safely to disk\n"));
 	printf(_("  -P, --progress                 write progress messages\n"));
+	printf(_("  -r, --use-postgresql-conf      use restore_command in the postgresql.conf to\n"));
+	printf(_("                                 retreive WALs from archive\n"));
+	printf(_("  -R, --restore-command=COMMAND  restore_command\n"));
 	printf(_("      --debug                    write a lot of debug messages\n"));
 	printf(_("  -V, --version                  output version information, then exit\n"));
 	printf(_("  -?, --help                     show this help, then exit\n"));
@@ -94,6 +101,8 @@ main(int argc, char **argv)
 		{"dry-run", no_argument, NULL, 'n'},
 		{"no-sync", no_argument, NULL, 'N'},
 		{"progress", no_argument, NULL, 'P'},
+		{"use-postgresql-conf", no_argument, NULL, 'r'},
+		{"restore-command", required_argument, NULL, 'R'},
 		{"debug", no_argument, NULL, 3},
 		{NULL, 0, NULL, 0}
 	};
@@ -129,7 +138,7 @@ main(int argc, char **argv)
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1)
+	while ((c = getopt_long(argc, argv, "D:nNPR:r", long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
@@ -141,6 +150,10 @@ main(int argc, char **argv)
 				showprogress = true;
 				break;
 
+			case 'r':
+				restore_wals = true;
+				break;
+
 			case 'n':
 				dry_run = true;
 				break;
@@ -157,6 +170,10 @@ main(int argc, char **argv)
 				datadir_target = pg_strdup(optarg);
 				break;
 
+			case 'R':
+				restore_command = pg_strdup(optarg);
+				break;
+
 			case 1:				/* --source-pgdata */
 				datadir_source = pg_strdup(optarg);
 				break;
@@ -223,6 +240,78 @@ main(int argc, char **argv)
 
 	umask(pg_mode_mask);
 
+	if (restore_command != NULL)
+	{
+		if (restore_wals)
+		{
+			fprintf(stderr, _("%s: conflicting options: both -r and -R are specified\n"),
+				progname);
+			fprintf(stderr, _("You must run %s with either -r/--use-postgresql-conf "
+							"or -R/--restore-command.\n"), progname);
+			exit(1);
+		}
+
+		pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command);
+	}
+	else if (restore_wals)
+	{
+		int   rc;
+		char  postgres_exec_path[MAXPGPATH],
+			  postgres_cmd[MAXPGPATH],
+			  cmd_output[MAX_RESTORE_COMMAND];
+		FILE *output_fp;
+
+		/* Find postgres executable. */
+		rc = find_other_exec(argv[0], "postgres",
+							 PG_BACKEND_VERSIONSTR,
+							 postgres_exec_path);
+
+		if (rc < 0)
+		{
+			char	full_path[MAXPGPATH];
+
+			if (find_my_exec(argv[0], full_path) < 0)
+				strlcpy(full_path, progname, sizeof(full_path));
+
+			if (rc == -1)
+				fprintf(stderr,
+						_("the program \"postgres\" is needed by %s "
+						"but was not found in the\n"
+						"same directory as \"%s\".\n"
+						"Check your installation.\n"),
+						progname, full_path);
+			else
+				fprintf(stderr,
+						_("the program \"postgres\" was found by \"%s\"\n"
+						"but was not the same version as %s.\n"
+						"Check your installation.\n"),
+						full_path, progname);
+			exit(1);
+		}
+
+		/* Build a command to execute for restore_command GUC retrieval if set. */
+		snprintf(postgres_cmd, sizeof(postgres_cmd), "%s -D %s -C restore_command",
+				 postgres_exec_path, datadir_target);
+
+		if ((output_fp = popen(postgres_cmd, "r")) == NULL ||
+			fgets(cmd_output, sizeof(cmd_output), output_fp) == NULL)
+			pg_fatal("could not get restore_command using %s: %s\n",
+					postgres_cmd, strerror(errno));
+
+		pclose(output_fp);
+
+		/* Remove trailing newline */
+		if (strchr(cmd_output, '\n') != NULL)
+			*strchr(cmd_output, '\n') = '\0';
+
+		if (!strcmp(cmd_output, ""))
+			pg_fatal("restore_command is not set on the target cluster\n");
+
+		restore_command = pg_strdup(cmd_output);
+
+		pg_log(PG_DEBUG, "using config variable restore_command=\'%s\'.\n", restore_command);
+	}
+
 	/* Connect to remote server */
 	if (connstr_source)
 		libpqConnect(connstr_source);
@@ -294,9 +383,8 @@ main(int argc, char **argv)
 		exit(0);
 	}
 
-	findLastCheckpoint(datadir_target, divergerec,
-					   lastcommontliIndex,
-					   &chkptrec, &chkpttli, &chkptredo);
+	findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex,
+					   &chkptrec, &chkpttli, &chkptredo, restore_command);
 	printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"),
 		   (uint32) (chkptrec >> 32), (uint32) chkptrec,
 		   chkpttli);
@@ -319,7 +407,7 @@ main(int argc, char **argv)
 	 */
 	pg_log(PG_PROGRESS, "reading WAL in target\n");
 	extractPageMap(datadir_target, chkptrec, lastcommontliIndex,
-				   ControlFile_target.checkPoint);
+				   ControlFile_target.checkPoint, restore_command);
 	filemap_finalize();
 
 	if (showprogress)
diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h
index 83b2898b8b..08a753475c 100644
--- a/src/bin/pg_rewind/pg_rewind.h
+++ b/src/bin/pg_rewind/pg_rewind.h
@@ -32,11 +32,10 @@ extern int	targetNentries;
 
 /* in parsexlog.c */
 extern void extractPageMap(const char *datadir, XLogRecPtr startpoint,
-			   int tliIndex, XLogRecPtr endpoint);
+			   int tliIndex, XLogRecPtr endpoint, const char *restoreCommand);
 extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr,
-				   int tliIndex,
-				   XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
-				   XLogRecPtr *lastchkptredo);
+				   int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
+				   XLogRecPtr *lastchkptredo, const char *restoreCommand);
 extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr,
 			  int tliIndex);
 
diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl
index 115192170e..8a6fa33016 100644
--- a/src/bin/pg_rewind/t/001_basic.pl
+++ b/src/bin/pg_rewind/t/001_basic.pl
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use TestLib;
-use Test::More tests => 10;
+use Test::More tests => 20;
 
 use FindBin;
 use lib $FindBin::RealBin;
@@ -106,5 +106,7 @@ in master, before promotion
 # Run the test in both modes
 run_test('local');
 run_test('remote');
+run_test('archive');
+run_test('archive_conf');
 
 exit(0);
diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.pl
index 6dc05720a1..99ea821cfe 100644
--- a/src/bin/pg_rewind/t/002_databases.pl
+++ b/src/bin/pg_rewind/t/002_databases.pl
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use TestLib;
-use Test::More tests => 6;
+use Test::More tests => 12;
 
 use FindBin;
 use lib $FindBin::RealBin;
@@ -62,5 +62,7 @@ template1
 # Run the test in both modes.
 run_test('local');
 run_test('remote');
+run_test('archive');
+run_test('archive_conf');
 
 exit(0);
diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.pl
index c4040bd562..24cec256de 100644
--- a/src/bin/pg_rewind/t/003_extrafiles.pl
+++ b/src/bin/pg_rewind/t/003_extrafiles.pl
@@ -3,7 +3,7 @@
 use strict;
 use warnings;
 use TestLib;
-use Test::More tests => 4;
+use Test::More tests => 8;
 
 use File::Find;
 
@@ -90,5 +90,7 @@ sub run_test
 # Run the test in both modes.
 run_test('local');
 run_test('remote');
+run_test('archive');
+run_test('archive_conf');
 
 exit(0);
diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm
index 85cae7e47b..4560fe4430 100644
--- a/src/bin/pg_rewind/t/RewindTest.pm
+++ b/src/bin/pg_rewind/t/RewindTest.pm
@@ -39,7 +39,9 @@ use Carp;
 use Config;
 use Exporter 'import';
 use File::Copy;
-use File::Path qw(rmtree);
+use File::Glob ':bsd_glob';
+use File::Path qw(remove_tree make_path);
+use File::Spec::Functions qw(catdir catfile);
 use IPC::Run qw(run);
 use PostgresNode;
 use TestLib;
@@ -197,6 +199,38 @@ sub promote_standby
 	return;
 }
 
+# Moves WAL files to the temporary location and returns restore_command
+# to get them back.
+sub move_wals
+{
+	my $tmp_dir = shift;
+	my $master_pgdata = shift;
+	my $wals_archive_dir = catdir($tmp_dir, "master_wals_archive");
+	my @wal_files = bsd_glob catfile($master_pgdata, "pg_wal", "0000000*");
+	my $restore_command;
+
+	remove_tree($wals_archive_dir);
+	make_path($wals_archive_dir) or die;
+
+	# Move all old master WAL files to the archive.
+	# Old master should be stopped at this point.
+	foreach my $wal_file (@wal_files)
+	{
+		move($wal_file, "$wals_archive_dir") or die;
+	}
+
+	if ($windows_os)
+	{
+		$restore_command = "copy $wals_archive_dir\\\%f \%p";
+	}
+	else
+	{
+		$restore_command = "cp $wals_archive_dir/\%f \%p";
+	}
+
+	return $restore_command;
+}
+
 sub run_pg_rewind
 {
 	my $test_mode       = shift;
@@ -249,6 +283,54 @@ sub run_pg_rewind
 			],
 			'pg_rewind remote');
 	}
+	elsif ($test_mode eq "archive")
+	{
+
+		# Do rewind using a local pgdata as source and
+		# specified directory with target WALs archive.
+		my $restore_command = move_wals($tmp_folder, $master_pgdata);
+
+		# Stop the new master and be ready to perform the rewind.
+		$node_standby->stop;
+
+		command_ok(
+			[
+				'pg_rewind',
+				"--debug",
+				"--source-pgdata=$standby_pgdata",
+				"--target-pgdata=$master_pgdata",
+				"--no-sync",
+				"--restore-command=$restore_command"
+			],
+			'pg_rewind archive');
+	}
+	elsif ($test_mode eq "archive_conf")
+	{
+
+		# Do rewind using a local pgdata as source and
+		# specified directory with target WALs archive.
+		my $master_conf_path = catfile($master_pgdata, 'postgresql.conf');
+		my $restore_command = move_wals($tmp_folder, $master_pgdata);
+
+		# Stop the new master and be ready to perform the rewind.
+		$node_standby->stop;
+
+		# Add restore_command to postgresql.conf of target cluster.
+		open(my $conf_fd, ">>", $master_conf_path) or die;
+		print $conf_fd "\nrestore_command='$restore_command'";
+		close $conf_fd;
+
+		command_ok(
+			[
+				'pg_rewind',
+				"--debug",
+				"--source-pgdata=$standby_pgdata",
+				"--target-pgdata=$master_pgdata",
+				"--no-sync",
+				"-r"
+			],
+			'pg_rewind archive_conf');
+	}
 	else
 	{

base-commit: 41531e42d34f4aca117d343b5e40f3f757dec5fe
-- 
2.17.1

--------------F3245A8D605DB552C5CDF408--