[Alexey Kondratov <a.kondratov@postgrespro.ru>]

Hi hackers,

Currently Postgres has options for continuous WAL files archiving, which 
is quite often used along with master-replica setup. OK, then the worst 
is happened and it's time to get your old master back and synchronize it 
with new master (ex-replica) with pg_rewind. However, required WAL files 
may be already archived and pg_rewind will fail. You can copy these 
files manually, but it is difficult to calculate, which ones you need. 
Anyway, it complicates building failover system with automatic failure 
recovery.

I expect, that it will be a good idea to allow pg_rewind to look for a 
restore_command in the target data directory recovery.conf or pass it is 
as a command line argument. Then pg_rewind can use it to get missing WAL 
files from the archive. I had a few talks with DBAs and came to 
conclusion, that this is a highly requested feature.

I prepared a proof of concept patch (please, find attached), which does 
exactly what I described above. I played with it a little and it seems 
to be working, tests were accordingly updated to verify this archive 
retrieval functionality too.

Patch is relatively simple excepting the one part: if we want to parse 
recovery.conf (with all possible includes, etc.) and get 
restore_command, then we should use guc-file.l parser, which is heavily 
linked to backend, e.g. in error reporting part. So I copied it and made 
frontend-safe version guc-file-fe.l. Personally, I don't think it's a 
good idea, but nothing else came to mind. It is also possible to leave 
the only one option -- passing restore_command as command line argument.

What do you think?


--

Alexey Kondratov

Postgres Professional: https://www.postgrespro.com

Russian Postgres Company

Attachments:

  [text/x-patch] pg_rewind-restore_command-v1.0.patch (37.9K, ../a3acff50-5a0d-9a2c-b3b2-ee36168955c1@postgrespro.ru/3-pg_rewind-restore_command-v1.0.patch)
  download | inline diff:
diff --combined src/bin/pg_rewind/Makefile
index a22fef1352,2bcfcc61af..0000000000
--- a/src/bin/pg_rewind/Makefile
+++ b/src/bin/pg_rewind/Makefile
@@@ -20,7 -20,6 +20,7 @@@ LDFLAGS_INTERNAL += $(libpq_pgport
  
  OBJS	= pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \
  	fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o logging.o \
 +	guc-file-fe.o \
  	$(WIN32RES)
  
  EXTRA_CLEAN = xlogreader.c
diff --combined src/bin/pg_rewind/RewindTest.pm
index 8dc39dbc05,1dce56d035..0000000000
--- a/src/bin/pg_rewind/RewindTest.pm
+++ b/src/bin/pg_rewind/RewindTest.pm
@@@ -40,7 -40,6 +40,7 @@@ use Config
  use Exporter 'import';
  use File::Copy;
  use File::Path qw(rmtree);
 +use File::Glob;
  use IPC::Run qw(run);
  use PostgresNode;
  use TestLib;
@@@ -249,41 -248,6 +249,41 @@@ sub run_pg_rewin
  				"--no-sync"
  			],
  			'pg_rewind remote');
 +	}
 +	elsif ($test_mode eq "archive")
 +	{
 +
 +		# Do rewind using a local pgdata as source and
 +		# specified directory with target WALs archive.
 +		my $wals_archive_dir = "${TestLib::tmp_check}/master_wals_archive";
 +		my $test_master_datadir = $node_master->data_dir;
 +		my @wal_files = glob "$test_master_datadir/pg_wal/0000000*";
 +		my $restore_command;
 +
 +		rmtree($wals_archive_dir);
 +		mkdir($wals_archive_dir) or die;
 +
 +		# Move all old master WAL files to the archive.
 +		# Old master should be stopped at this point.
 +		foreach my $wal_file (@wal_files)
 +		{
 +			move($wal_file, "$wals_archive_dir/") or die;
 +		}
 +
 +		$restore_command = "cp $wals_archive_dir/\%f \%p";
 +
 +		# Stop the new master and be ready to perform the rewind.
 +		$node_standby->stop;
 +		command_ok(
 +			[
 +				'pg_rewind',
 +				"--debug",
 +				"--source-pgdata=$standby_pgdata",
 +				"--target-pgdata=$master_pgdata",
 +				"--no-sync",
 +				"-R", $restore_command
 +			],
 +			'pg_rewind archive');
  	}
  	else
  	{
diff --combined src/bin/pg_rewind/parsexlog.c
index 11a9c26cd2,40028471bf..0000000000
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@@ -12,7 -12,6 +12,7 @@@
  #include "postgres_fe.h"
  
  #include <unistd.h>
 +#include <sys/stat.h>
  
  #include "pg_rewind.h"
  #include "filemap.h"
@@@ -46,10 -45,7 +46,10 @@@ static char xlogfpath[MAXPGPATH]
  typedef struct XLogPageReadPrivate
  {
  	const char *datadir;
 +	const char *restoreCommand;
  	int			tliIndex;
 +	XLogRecPtr  oldrecptr;
 +	TimeLineID  oldtli;
  } XLogPageReadPrivate;
  
  static int SimpleXLogPageRead(XLogReaderState *xlogreader,
@@@ -57,10 -53,6 +57,10 @@@
  				   int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
  				   TimeLineID *pageTLI);
  
 +static bool RestoreArchivedWAL(const char *path, const char *xlogfname, 
 +					off_t expectedSize, const char *restoreCommand,
 +					const char *lastRestartPointFname);
 +
  /*
   * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
   * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
@@@ -68,19 -60,15 +68,19 @@@
   */
  void
  extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 -			   XLogRecPtr endpoint)
 +			   ControlFileData *targetCF, const char *restore_command)
  {
  	XLogRecord *record;
 +	XLogRecPtr endpoint = targetCF->checkPoint;
  	XLogReaderState *xlogreader;
  	char	   *errormsg;
  	XLogPageReadPrivate private;
  
  	private.datadir = datadir;
  	private.tliIndex = tliIndex;
 +	private.restoreCommand = restore_command;
 +	private.oldrecptr = targetCF->checkPointCopy.redo;
 +	private.oldtli = targetCF->checkPointCopy.ThisTimeLineID;
  	xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
  									&private);
  	if (xlogreader == NULL)
@@@ -166,9 -154,9 +166,9 @@@ readOneRecord(const char *datadir, XLog
   * Find the previous checkpoint preceding given WAL location.
   */
  void
 -findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 +findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr forkptr, int tliIndex,
  				   XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
 -				   XLogRecPtr *lastchkptredo)
 +				   XLogRecPtr *lastchkptredo, const char *restoreCommand)
  {
  	/* Walk backwards, starting from the given record */
  	XLogRecord *record;
@@@ -193,9 -181,6 +193,9 @@@
  
  	private.datadir = datadir;
  	private.tliIndex = tliIndex;
 +	private.restoreCommand = restoreCommand;
 +	private.oldrecptr = targetCF->checkPointCopy.redo;
 +	private.oldtli = targetCF->checkPointCopy.ThisTimeLineID;
  	xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
  									&private);
  	if (xlogreader == NULL)
@@@ -306,53 -291,9 +306,53 @@@ SimpleXLogPageRead(XLogReaderState *xlo
  
  		if (xlogreadfd < 0)
  		{
 -			printf(_("could not open file \"%s\": %s\n"), xlogfpath,
 +			bool  restore_ok;
 +			char  lastRestartPointFname[MAXFNAMELEN];
 +			XLogSegNo restartSegNo;
 +
 +			/*
 +			 * If we have no restore_command to execute, then exit.
 +			 */
 +			if (private->restoreCommand == NULL)
 +			{
 +				printf(_("could not open file \"%s\": %s\n"), xlogfpath,
  				   strerror(errno));
 -			return -1;
 +				return -1;
 +			}
 +			
 +			XLByteToSeg(private->oldrecptr, restartSegNo, WalSegSz);
 +			XLogFileName(lastRestartPointFname, private->oldtli, restartSegNo,
 +						 WalSegSz);
 +
 +			/*
 +			 * Since we have restore_command to execute, then try to retreive
 +			 * missing WAL file from the archive.
 +			 */
 +			restore_ok = RestoreArchivedWAL(private->datadir,
 +											xlogfname,
 +											WalSegSz,
 +											private->restoreCommand,
 +											lastRestartPointFname);
 +
 +			if (restore_ok)
 +			{
 +				xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
 +
 +				if (xlogreadfd < 0)
 +				{
 +					printf(_("could not open restored from archive file \"%s\": %s\n"), xlogfpath,
 +				    	   strerror(errno));
 +				    return -1;
 +				}
 +				else
 +					pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n", xlogfpath);
 +			}
 +			else
 +			{
 +				printf(_("could not restore file \"%s\" from archive: %s\n"), xlogfname,
 +					   strerror(errno));
 +				return -1;
 +			}
  		}
  	}
  
@@@ -468,116 -409,3 +468,116 @@@ extractPageInfo(XLogReaderState *record
  		process_block_change(forknum, rnode, blkno);
  	}
  }
 +
 +/*
 + * Attempt to retrieve the specified file from off-line archival storage.
 + * If successful return true.
 + *
 + * For fixed-size files, the caller may pass the expected size as an
 + * additional crosscheck on successful recovery. If the file size is not
 + * known, set expectedSize = 0.
 + * 
 + * This is an adapted to frontend version of
 + * RestoreArchivedFile function from transam/xlogarchive.c
 + */
 +bool
 +RestoreArchivedWAL(const char *path, const char *xlogfname, off_t expectedSize,
 +				   const char *restoreCommand, const char *lastRestartPointFname)
 +{
 +	char		xlogpath[MAXPGPATH];
 +	char		xlogRestoreCmd[MAXPGPATH];
 +	char	   *dp;
 +	char	   *endp;
 +	const char *sp;
 +	int			rc;
 +	struct stat stat_buf;
 +
 +	snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);
 +
 +	/*
 +	 * Construct the command to be executed.
 +	 */
 +	dp = xlogRestoreCmd;
 +	endp = xlogRestoreCmd + MAXPGPATH - 1;
 +	*endp = '\0';
 +
 +	for (sp = restoreCommand; *sp; sp++)
 +	{
 +		if (*sp == '%')
 +		{
 +			switch (sp[1])
 +			{
 +				case 'p':
 +					/* %p: relative path of target file */
 +					sp++;
 +					StrNCpy(dp, xlogpath, endp - dp);
 +					make_native_path(dp);
 +					dp += strlen(dp);
 +					break;
 +				case 'f':
 +					/* %f: filename of desired file */
 +					sp++;
 +					StrNCpy(dp, xlogfname, endp - dp);
 +					dp += strlen(dp);
 +					break;
 +				case 'r':
 +					/* %r: filename of last restartpoint */
 +					sp++;
 +					StrNCpy(dp, lastRestartPointFname, endp - dp);
 +					dp += strlen(dp);
 +					break;
 +				case '%':
 +					/* convert %% to a single % */
 +					sp++;
 +					if (dp < endp)
 +						*dp++ = *sp;
 +					break;
 +				default:
 +					/* otherwise treat the % as not special */
 +					if (dp < endp)
 +						*dp++ = *sp;
 +					break;
 +			}
 +		}
 +		else
 +		{
 +			if (dp < endp)
 +				*dp++ = *sp;
 +		}
 +	}
 +	*dp = '\0';
 +
 +	/*
 +	 * Execute restore_command, which should copy
 +	 * the missing WAL file from archival storage.
 +	 */
 +	rc = system(xlogRestoreCmd);
 +
 +	if (rc == 0)
 +	{
 +		/*
 +		 * Command apparently succeeded, but let's make sure the file is
 +		 * really there now and has the correct size.
 +		 */
 +		if (stat(xlogpath, &stat_buf) == 0)
 +		{
 +			if (expectedSize > 0 && stat_buf.st_size != expectedSize)
 +			{
 +				printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"),
 +					   	xlogfname, (unsigned long) stat_buf.st_size, (unsigned long) expectedSize,
 +					   	strerror(errno));
 +			}
 +			else
 +				return true;
 +		}
 +		else
 +		{
 +			/* Stat failed */
 +			printf(_("could not stat file \"%s\": %s"),
 +				   	xlogpath,
 +					strerror(errno));
 +		}
 +	}
 +
 +	return false;
 +}
diff --combined src/bin/pg_rewind/pg_rewind.c
index 3acac817eb,9653106386..0000000000
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@@ -19,13 -19,11 +19,13 @@@
  #include "file_ops.h"
  #include "filemap.h"
  #include "logging.h"
 +#include "guc-file-fe.h"
  
  #include "access/timeline.h"
  #include "access/xlog_internal.h"
  #include "catalog/catversion.h"
  #include "catalog/pg_control.h"
 +#include "common/controldata_utils.h"
  #include "common/file_perm.h"
  #include "common/file_utils.h"
  #include "common/restricted_token.h"
@@@ -54,13 -52,11 +54,13 @@@ int			WalSegSz
  char	   *datadir_target = NULL;
  char	   *datadir_source = NULL;
  char	   *connstr_source = NULL;
 +char	   *restore_command = NULL;
  
  bool		debug = false;
  bool		showprogress = false;
  bool		dry_run = false;
  bool		do_sync = true;
 +bool		restore_wals = false;
  
  /* Target history */
  TimeLineHistoryEntry *targetHistory;
@@@ -79,9 -75,6 +79,9 @@@ usage(const char *progname
  	printf(_("  -N, --no-sync                  do not wait for changes to be written\n"));
  	printf(_("                                 safely to disk\n"));
  	printf(_("  -P, --progress                 write progress messages\n"));
 +	printf(_("  -r, --use-recovery-conf        use restore_command in the recovery.conf to\n"));
 +	printf(_("                                 retreive WALs from archive\n"));
 +	printf(_("  -R, --restore-command=COMMAND  restore command\n"));
  	printf(_("      --debug                    write a lot of debug messages\n"));
  	printf(_("  -V, --version                  output version information, then exit\n"));
  	printf(_("  -?, --help                     show this help, then exit\n"));
@@@ -101,12 -94,9 +101,12 @@@ main(int argc, char **argv
  		{"dry-run", no_argument, NULL, 'n'},
  		{"no-sync", no_argument, NULL, 'N'},
  		{"progress", no_argument, NULL, 'P'},
 +		{"use-recovery-conf", no_argument, NULL, 'r'},
 +		{"restore-command", required_argument, NULL, 'R'},
  		{"debug", no_argument, NULL, 3},
  		{NULL, 0, NULL, 0}
  	};
 +	char		recfile_fullpath[MAXPGPATH];
  	int			option_index;
  	int			c;
  	XLogRecPtr	divergerec;
@@@ -139,7 -129,7 +139,7 @@@
  		}
  	}
  
 -	while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1)
 +	while ((c = getopt_long(argc, argv, "DR:nNPr", long_options, &option_index)) != -1)
  	{
  		switch (c)
  		{
@@@ -150,10 -140,6 +150,10 @@@
  			case 'P':
  				showprogress = true;
  				break;
 +			
 +			case 'r':
 +				restore_wals = true;
 +				break;
  
  			case 'n':
  				dry_run = true;
@@@ -171,10 -157,6 +171,10 @@@
  				datadir_target = pg_strdup(optarg);
  				break;
  
 +			case 'R':
 +				restore_command = pg_strdup(optarg);
 +				break;
 +
  			case 1:				/* --source-pgdata */
  				datadir_source = pg_strdup(optarg);
  				break;
@@@ -241,71 -223,6 +241,71 @@@
  
  	umask(pg_mode_mask);
  
 +	if (restore_command != NULL)
 +	{
 +		pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command);
 +	}
 +	else if (restore_wals)
 +	{
 +		FILE	   *recovery_conf_file;
 +
 +		/* 
 +		 * Look for recovery.conf in the target data directory and
 +		 * try to get restore_command from there.
 +		 */
 +		snprintf(recfile_fullpath, sizeof(recfile_fullpath), "%s/%s", datadir_target, RECOVERY_COMMAND_FILE);
 +		recovery_conf_file = fopen(recfile_fullpath, "r");
 +
 +		if (recovery_conf_file == NULL)
 +		{
 +			fprintf(stderr, _("%s: option -r/--use-recovery-conf is specified, but recovery.conf is absent in the target directory\n"),
 +					progname);
 +			fprintf(stdout, _("You have to add recovery.conf or pass restore_command with -R/--restore-command option.\n"));
 +			exit(1);
 +		}
 +		else
 +		{
 +			ConfigVariable *item,
 +						   *head = NULL,
 +						   *tail = NULL;
 +			bool			config_is_parsed;
 +
 +			/*
 +			 * We pass a fullpath to the recovery.conf as calling_file here, since
 +			 * parser will use its parent directory as base for all further includes
 +			 * if any exist.
 +			 */
 +			config_is_parsed = ParseConfigFile(RECOVERY_COMMAND_FILE, true,
 +											   recfile_fullpath, 0, 0,
 +											   PG_WARNING, &head, &tail);
 +			fclose(recovery_conf_file);
 +
 +			if (config_is_parsed)
 +			{
 +				for (item = head; item; item = item->next)
 +				{
 +					if (strcmp(item->name, "restore_command") == 0)
 +					{
 +						if (restore_command != NULL)
 +						{
 +							pfree(restore_command);
 +							restore_command = NULL;
 +						}
 +						restore_command = pstrdup(item->value);
 +						pg_log(PG_DEBUG, "using restore_command=\'%s\' from recovery.conf.\n", restore_command);
 +					}
 +				}
 +
 +				if (restore_command == NULL)
 +					pg_fatal("could not find restore_command in recovery.conf file %s\n", recfile_fullpath);
 +			}
 +			else
 +				pg_fatal("could not parse recovery.conf file %s\n", recfile_fullpath);
 +
 +			FreeConfigVariables(head);
 +		}
 +	}
 +
  	/* Connect to remote server */
  	if (connstr_source)
  		libpqConnect(connstr_source);
@@@ -377,9 -294,9 +377,9 @@@
  		exit(0);
  	}
  
 -	findLastCheckpoint(datadir_target, divergerec,
 +	findLastCheckpoint(datadir_target, &ControlFile_target, divergerec,
  					   lastcommontliIndex,
 -					   &chkptrec, &chkpttli, &chkptredo);
 +					   &chkptrec, &chkpttli, &chkptredo, restore_command);
  	printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"),
  		   (uint32) (chkptrec >> 32), (uint32) chkptrec,
  		   chkpttli);
@@@ -402,7 -319,7 +402,7 @@@
  	 */
  	pg_log(PG_PROGRESS, "reading WAL in target\n");
  	extractPageMap(datadir_target, chkptrec, lastcommontliIndex,
 -				   ControlFile_target.checkPoint);
 +				   &ControlFile_target, restore_command);
  	filemap_finalize();
  
  	if (showprogress)
diff --combined src/bin/pg_rewind/pg_rewind.h
index ba9d13e9bd,3f4ba7a267..0000000000
--- a/src/bin/pg_rewind/pg_rewind.h
+++ b/src/bin/pg_rewind/pg_rewind.h
@@@ -14,7 -14,6 +14,7 @@@
  #include "datapagemap.h"
  
  #include "access/timeline.h"
 +#include "catalog/pg_control.h"
  #include "storage/block.h"
  #include "storage/relfilenode.h"
  
@@@ -33,10 -32,11 +33,10 @@@ extern int	targetNentries
  
  /* in parsexlog.c */
  extern void extractPageMap(const char *datadir, XLogRecPtr startpoint,
 -			   int tliIndex, XLogRecPtr endpoint);
 -extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr,
 -				   int tliIndex,
 -				   XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
 -				   XLogRecPtr *lastchkptredo);
 +			   int tliIndex, ControlFileData *targetCF, const char *restoreCommand);
 +extern void findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr searchptr,
 +				   int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
 +				   XLogRecPtr *lastchkptredo, const char *restoreCommand);
  extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr,
  			  int tliIndex);
  
diff --combined src/bin/pg_rewind/t/001_basic.pl
index 22777bff82,53dbf45be2..0000000000
--- a/src/bin/pg_rewind/t/001_basic.pl
+++ b/src/bin/pg_rewind/t/001_basic.pl
@@@ -1,7 -1,7 +1,7 @@@
  use strict;
  use warnings;
  use TestLib;
 -use Test::More tests => 10;
 +use Test::More tests => 15;
  
  use RewindTest;
  
@@@ -103,6 -103,5 +103,6 @@@ in master, before promotio
  # Run the test in both modes
  run_test('local');
  run_test('remote');
 +run_test('archive');
  
  exit(0);
diff --combined src/bin/pg_rewind/t/002_databases.pl
index 10c835efc1,2c9e427831..0000000000
--- a/src/bin/pg_rewind/t/002_databases.pl
+++ b/src/bin/pg_rewind/t/002_databases.pl
@@@ -1,7 -1,7 +1,7 @@@
  use strict;
  use warnings;
  use TestLib;
 -use Test::More tests => 6;
 +use Test::More tests => 9;
  
  use RewindTest;
  
@@@ -59,6 -59,5 +59,6 @@@ template
  # Run the test in both modes.
  run_test('local');
  run_test('remote');
 +run_test('archive');
  
  exit(0);
diff --combined src/bin/pg_rewind/t/003_extrafiles.pl
index f94ce329f4,496f38c457..0000000000
--- a/src/bin/pg_rewind/t/003_extrafiles.pl
+++ b/src/bin/pg_rewind/t/003_extrafiles.pl
@@@ -3,7 -3,7 +3,7 @@@
  use strict;
  use warnings;
  use TestLib;
 -use Test::More tests => 4;
 +use Test::More tests => 6;
  
  use File::Find;
  
@@@ -87,6 -87,5 +87,6 @@@ sub run_tes
  # Run the test in both modes.
  run_test('local');
  run_test('remote');
 +run_test('archive');
  
  exit(0);
diff --git a/src/bin/pg_rewind/guc-file-fe.h b/src/bin/pg_rewind/guc-file-fe.h
new file mode 100644
index 0000000000..cf480b806a
--- /dev/null
+++ b/src/bin/pg_rewind/guc-file-fe.h
@@ -0,0 +1,40 @@
+#ifndef PG_REWIND_GUC_FILE_FE_H
+#define PG_REWIND_GUC_FILE_FE_H
+
+#include "c.h"
+
+#define RECOVERY_COMMAND_FILE	"recovery.conf"
+
+/*
+ * Parsing the configuration file(s) will return a list of name-value pairs
+ * with source location info.  We also abuse this data structure to carry
+ * error reports about the config files.  An entry reporting an error will
+ * have errmsg != NULL, and might have NULLs for name, value, and/or filename.
+ *
+ * If "ignore" is true, don't attempt to apply the item (it might be an error
+ * report, or an item we determined to be duplicate).  "applied" is set true
+ * if we successfully applied, or could have applied, the setting.
+ */
+typedef struct ConfigVariable
+{
+	char	   *name;
+	char	   *value;
+	char	   *errmsg;
+	char	   *filename;
+	int			sourceline;
+	bool		ignore;
+	bool		applied;
+	struct ConfigVariable *next;
+} ConfigVariable;
+
+extern bool ParseConfigFile(const char *config_file, bool strict,
+				const char *calling_file, int calling_lineno,
+				int depth, int elevel,
+				ConfigVariable **head_p, ConfigVariable **tail_p);
+
+extern bool ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,
+				ConfigVariable **head_p, ConfigVariable **tail_p);
+
+extern void FreeConfigVariables(ConfigVariable *list);
+
+#endif							/* PG_REWIND_GUC_FILE_FE_H */
diff --git a/src/bin/pg_rewind/guc-file-fe.l b/src/bin/pg_rewind/guc-file-fe.l
new file mode 100644
index 0000000000..13882191ba
--- /dev/null
+++ b/src/bin/pg_rewind/guc-file-fe.l
@@ -0,0 +1,774 @@
+/* -*-pgsql-c-*- */
+/*
+ * Configuration files parser for usage in frontend.
+ *
+ * Copyright (c) 2000-2018, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_rewind/guc-file-fe.l
+ * Modified version of src/backend/utils/misc/guc-file.l
+ */
+
+%{
+
+#include <ctype.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <setjmp.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "guc-file-fe.h"
+#include "logging.h"
+
+#include "common/fe_memutils.h"
+
+/*
+ * flex emits a yy_fatal_error() function that it calls in response to
+ * critical errors like malloc failure, file I/O errors, and detection of
+ * internal inconsistency.  That function prints a message and calls exit().
+ * Mutate it to instead call our handler, which jumps out of the parser.
+ */
+#undef fprintf
+#define fprintf(file, fmt, msg) GUC_flex_fatal(msg)
+
+enum
+{
+	GUC_ID = 1,
+	GUC_STRING = 2,
+	GUC_INTEGER = 3,
+	GUC_REAL = 4,
+	GUC_EQUALS = 5,
+	GUC_UNQUOTED_STRING = 6,
+	GUC_QUALIFIED_ID = 7,
+	GUC_EOL = 99,
+	GUC_ERROR = 100
+};
+
+static unsigned int ConfigFileLineno;
+static const char *GUC_flex_fatal_errmsg;
+static sigjmp_buf *GUC_flex_fatal_jmp;
+
+static void FreeConfigVariable(ConfigVariable *item);
+
+static void record_config_file_error(const char *errmsg,
+						 const char *config_file,
+						 int lineno,
+						 ConfigVariable **head_p,
+						 ConfigVariable **tail_p);
+
+static int	GUC_flex_fatal(const char *msg);
+static char *GUC_scanstr(const char *s);
+static bool ParseConfigDirectory(const char *includedir,
+					 const char *calling_file, int calling_lineno,
+					 int depth, int elevel,
+					 ConfigVariable **head_p,
+					 ConfigVariable **tail_p);
+
+/* LCOV_EXCL_START */
+
+%}
+
+%option 8bit
+%option never-interactive
+%option nodefault
+%option noinput
+%option nounput
+%option noyywrap
+%option warn
+%option prefix="GUC_yy"
+
+
+SIGN			("-"|"+")
+DIGIT			[0-9]
+HEXDIGIT		[0-9a-fA-F]
+
+UNIT_LETTER		[a-zA-Z]
+
+INTEGER			{SIGN}?({DIGIT}+|0x{HEXDIGIT}+){UNIT_LETTER}*
+
+EXPONENT		[Ee]{SIGN}?{DIGIT}+
+REAL			{SIGN}?{DIGIT}*"."{DIGIT}*{EXPONENT}?
+
+LETTER			[A-Za-z_\200-\377]
+LETTER_OR_DIGIT [A-Za-z_0-9\200-\377]
+
+ID				{LETTER}{LETTER_OR_DIGIT}*
+QUALIFIED_ID	{ID}"."{ID}
+
+UNQUOTED_STRING {LETTER}({LETTER_OR_DIGIT}|[-._:/])*
+STRING			\'([^'\\\n]|\\.|\'\')*\'
+
+%%
+
+\n				ConfigFileLineno++; return GUC_EOL;
+[ \t\r]+		/* eat whitespace */
+#.*				/* eat comment (.* matches anything until newline) */
+
+{ID}			return GUC_ID;
+{QUALIFIED_ID}	return GUC_QUALIFIED_ID;
+{STRING}		return GUC_STRING;
+{UNQUOTED_STRING} return GUC_UNQUOTED_STRING;
+{INTEGER}		return GUC_INTEGER;
+{REAL}			return GUC_REAL;
+=				return GUC_EQUALS;
+
+.				return GUC_ERROR;
+
+%%
+
+/* LCOV_EXCL_STOP */
+
+/*
+ * the bare comparison function for GUC names
+ */
+static int
+guc_name_compare(const char *namea, const char *nameb)
+{
+	/*
+	 * The temptation to use strcasecmp() here must be resisted, because the
+	 * array ordering has to remain stable across setlocale() calls. So, build
+	 * our own with a simple ASCII-only downcasing.
+	 */
+	while (*namea && *nameb)
+	{
+		char		cha = *namea++;
+		char		chb = *nameb++;
+
+		if (cha >= 'A' && cha <= 'Z')
+			cha += 'a' - 'A';
+		if (chb >= 'A' && chb <= 'Z')
+			chb += 'a' - 'A';
+		if (cha != chb)
+			return cha - chb;
+	}
+	if (*namea)
+		return 1;				/* a is longer */
+	if (*nameb)
+		return -1;				/* b is longer */
+	return 0;
+}
+
+/*
+ * Given a configuration file or directory location that may be a relative
+ * path, return an absolute one.  We consider the location to be relative to
+ * the directory holding the calling file.
+ */
+static char *
+AbsoluteConfigLocation(const char *location, const char *calling_file)
+{
+	char abs_path[MAXPGPATH];
+
+	if (is_absolute_path(location))
+		return pstrdup(location);
+	else
+	{
+		strlcpy(abs_path, calling_file, sizeof(abs_path));
+		get_parent_directory(abs_path);
+		join_path_components(abs_path, abs_path, location);
+		canonicalize_path(abs_path);
+
+		return pstrdup(abs_path);
+	}
+}
+
+/*
+ * Read and parse a single configuration file.  This function recurses
+ * to handle "include" directives.
+ *
+ * If "strict" is true, treat failure to open the config file as an error,
+ * otherwise just skip the file.
+ *
+ * calling_file/calling_lineno identify the source of the request.
+ * Pass NULL/0 if not recursing from an inclusion request.
+ *
+ * See ParseConfigFp for further details.  This one merely adds opening the
+ * config file rather than working from a caller-supplied file descriptor,
+ * and absolute-ifying the path name if necessary.
+ */
+bool
+ParseConfigFile(const char *config_file, bool strict,
+				const char *calling_file, int calling_lineno,
+				int depth, int elevel,
+				ConfigVariable **head_p,
+				ConfigVariable **tail_p)
+{
+	char	   *abs_path;
+	bool		OK = true;
+	FILE	   *fp;
+
+	/*
+	 * Reject too-deep include nesting depth.  This is just a safety check to
+	 * avoid dumping core due to stack overflow if an include file loops back
+	 * to itself.  The maximum nesting depth is pretty arbitrary.
+	 */
+	if (depth > 10)
+	{
+		pg_log(PG_WARNING, 
+				"could not open configuration file \"%s\": maximum nesting depth exceeded\n",
+				config_file);
+		record_config_file_error("nesting depth exceeded",
+								 calling_file, calling_lineno,
+								 head_p, tail_p);
+		return false;
+	}
+
+	abs_path = AbsoluteConfigLocation(config_file, calling_file);
+	fp = fopen(abs_path, "r");
+	if (!fp)
+	{
+		if (strict)
+		{
+			pg_log(PG_WARNING, 
+				"could not open configuration file \"%s\"\n",
+				abs_path);
+			record_config_file_error(psprintf("could not open file \"%s\"",
+											  abs_path),
+									 calling_file, calling_lineno,
+									 head_p, tail_p);
+			OK = false;
+		}
+		else
+		{
+			pg_log(PG_WARNING, 
+				"skipping missing configuration file \"%s\"\n",
+				abs_path);
+		}
+		goto cleanup;
+	}
+
+	OK = ParseConfigFp(fp, abs_path, depth, elevel, head_p, tail_p);
+
+cleanup:
+	if (fp)
+		fclose(fp);
+	pfree(abs_path);
+
+	return OK;
+}
+
+/*
+ * Capture an error message in the ConfigVariable list returned by
+ * config file parsing.
+ */
+static void
+record_config_file_error(const char *errmsg,
+						 const char *config_file,
+						 int lineno,
+						 ConfigVariable **head_p,
+						 ConfigVariable **tail_p)
+{
+	ConfigVariable *item;
+
+	item = palloc(sizeof *item);
+	item->name = NULL;
+	item->value = NULL;
+	item->errmsg = pstrdup(errmsg);
+	item->filename = config_file ? pstrdup(config_file) : NULL;
+	item->sourceline = lineno;
+	item->ignore = true;
+	item->applied = false;
+	item->next = NULL;
+	if (*head_p == NULL)
+		*head_p = item;
+	else
+		(*tail_p)->next = item;
+	*tail_p = item;
+}
+
+/*
+ * Flex fatal errors bring us here.  Stash the error message and jump back to
+ * ParseConfigFp().  Assume all msg arguments point to string constants; this
+ * holds for flex 2.5.31 (earliest we support) and flex 2.5.35 (latest as of
+ * this writing).  Otherwise, we would need to copy the message.
+ *
+ * We return "int" since this takes the place of calls to fprintf().
+*/
+static int
+GUC_flex_fatal(const char *msg)
+{
+	GUC_flex_fatal_errmsg = msg;
+	siglongjmp(*GUC_flex_fatal_jmp, 1);
+	return 0;					/* keep compiler quiet */
+}
+
+/*
+ * Read and parse a single configuration file.  This function recurses
+ * to handle "include" directives.
+ *
+ * Input parameters:
+ *	fp: file pointer from AllocateFile for the configuration file to parse
+ *	config_file: absolute or relative path name of the configuration file
+ *	depth: recursion depth (should be 0 in the outermost call)
+ *	elevel: error logging level to use
+ * Input/Output parameters:
+ *	head_p, tail_p: head and tail of linked list of name/value pairs
+ *
+ * *head_p and *tail_p must be initialized, either to NULL or valid pointers
+ * to a ConfigVariable list, before calling the outer recursion level.  Any
+ * name-value pairs read from the input file(s) will be appended to the list.
+ * Error reports will also be appended to the list, if elevel < ERROR.
+ *
+ * Returns TRUE if successful, FALSE if an error occurred.
+ *
+ * Note: this function is used to parse not only postgresql.conf, but
+ * various other configuration files that use the same "name = value"
+ * syntax.  Hence, do not do anything here or in the subsidiary routines
+ * ParseConfigFile/ParseConfigDirectory that assumes we are processing
+ * GUCs specifically.
+ */
+bool
+ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,
+			  ConfigVariable **head_p, ConfigVariable **tail_p)
+{
+	volatile bool OK = true;
+	unsigned int save_ConfigFileLineno = ConfigFileLineno;
+	sigjmp_buf *save_GUC_flex_fatal_jmp = GUC_flex_fatal_jmp;
+	sigjmp_buf	flex_fatal_jmp;
+	volatile YY_BUFFER_STATE lex_buffer = NULL;
+	int			errorcount;
+	int			token;
+
+	if (sigsetjmp(flex_fatal_jmp, 1) == 0)
+		GUC_flex_fatal_jmp = &flex_fatal_jmp;
+	else
+	{
+		/*
+		 * Regain control after a fatal, internal flex error.  It may have
+		 * corrupted parser state.  Consequently, abandon the file, but trust
+		 * that the state remains sane enough for yy_delete_buffer().
+		 */
+		pg_log(PG_WARNING, "%s at file \"%s\" line %u\n", GUC_flex_fatal_errmsg,
+				config_file, ConfigFileLineno);
+		record_config_file_error(GUC_flex_fatal_errmsg,
+								 config_file, ConfigFileLineno,
+								 head_p, tail_p);
+		OK = false;
+		goto cleanup;
+	}
+
+	/*
+	 * Parse
+	 */
+	ConfigFileLineno = 1;
+	errorcount = 0;
+
+	lex_buffer = yy_create_buffer(fp, YY_BUF_SIZE);
+	yy_switch_to_buffer(lex_buffer);
+
+	/* This loop iterates once per logical line */
+	while ((token = yylex()))
+	{
+		char	   *opt_name = NULL;
+		char	   *opt_value = NULL;
+		ConfigVariable *item;
+
+		if (token == GUC_EOL)	/* empty or comment line */
+			continue;
+
+		/* first token on line is option name */
+		if (token != GUC_ID && token != GUC_QUALIFIED_ID)
+			goto parse_error;
+		opt_name = pstrdup(yytext);
+
+		/* next we have an optional equal sign; discard if present */
+		token = yylex();
+		if (token == GUC_EQUALS)
+			token = yylex();
+
+		/* now we must have the option value */
+		if (token != GUC_ID &&
+			token != GUC_STRING &&
+			token != GUC_INTEGER &&
+			token != GUC_REAL &&
+			token != GUC_UNQUOTED_STRING)
+			goto parse_error;
+		if (token == GUC_STRING)	/* strip quotes and escapes */
+			opt_value = GUC_scanstr(yytext);
+		else
+			opt_value = pstrdup(yytext);
+
+		/* now we'd like an end of line, or possibly EOF */
+		token = yylex();
+		if (token != GUC_EOL)
+		{
+			if (token != 0)
+				goto parse_error;
+			/* treat EOF like \n for line numbering purposes, cf bug 4752 */
+			ConfigFileLineno++;
+		}
+
+		/* OK, process the option name and value */
+		if (guc_name_compare(opt_name, "include_dir") == 0)
+		{
+			/*
+			 * An include_dir directive isn't a variable and should be
+			 * processed immediately.
+			 */
+			if (!ParseConfigDirectory(opt_value,
+									  config_file, ConfigFileLineno - 1,
+									  depth + 1, elevel,
+									  head_p, tail_p))
+				OK = false;
+			yy_switch_to_buffer(lex_buffer);
+			pfree(opt_name);
+			pfree(opt_value);
+		}
+		else if (guc_name_compare(opt_name, "include_if_exists") == 0)
+		{
+			/*
+			 * An include_if_exists directive isn't a variable and should be
+			 * processed immediately.
+			 */
+			if (!ParseConfigFile(opt_value, false,
+								 config_file, ConfigFileLineno - 1,
+								 depth + 1, elevel,
+								 head_p, tail_p))
+				OK = false;
+			yy_switch_to_buffer(lex_buffer);
+			pfree(opt_name);
+			pfree(opt_value);
+		}
+		else if (guc_name_compare(opt_name, "include") == 0)
+		{
+			/*
+			 * An include directive isn't a variable and should be processed
+			 * immediately.
+			 */
+			if (!ParseConfigFile(opt_value, true,
+								 config_file, ConfigFileLineno - 1,
+								 depth + 1, elevel,
+								 head_p, tail_p))
+				OK = false;
+			yy_switch_to_buffer(lex_buffer);
+			pfree(opt_name);
+			pfree(opt_value);
+		}
+		else
+		{
+			/* ordinary variable, append to list */
+			item = palloc(sizeof *item);
+			item->name = opt_name;
+			item->value = opt_value;
+			item->errmsg = NULL;
+			item->filename = pstrdup(config_file);
+			item->sourceline = ConfigFileLineno - 1;
+			item->ignore = false;
+			item->applied = false;
+			item->next = NULL;
+			if (*head_p == NULL)
+				*head_p = item;
+			else
+				(*tail_p)->next = item;
+			*tail_p = item;
+		}
+
+		/* break out of loop if read EOF, else loop for next line */
+		if (token == 0)
+			break;
+		continue;
+
+parse_error:
+		/* release storage if we allocated any on this line */
+		if (opt_name)
+			pfree(opt_name);
+		if (opt_value)
+			pfree(opt_value);
+
+		/* report the error */
+		if (token == GUC_EOL || token == 0)
+		{
+			pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near end of line\n",
+					config_file, ConfigFileLineno - 1);
+			record_config_file_error("syntax error",
+									 config_file, ConfigFileLineno - 1,
+									 head_p, tail_p);
+		}
+		else
+		{
+			pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near token \"%s\"\n",
+				config_file, ConfigFileLineno, yytext);
+			record_config_file_error("syntax error",
+									 config_file, ConfigFileLineno,
+									 head_p, tail_p);
+		}
+		OK = false;
+		errorcount++;
+
+		/*
+		 * To avoid producing too much noise when fed a totally bogus file,
+		 * give up after 100 syntax errors per file (an arbitrary number).
+		 */
+		if (errorcount >= 100)
+		{
+			pg_log(PG_WARNING, "too many syntax errors found, abandoning file \"%s\"\n",
+					config_file);
+			break;
+		}
+
+		/* resync to next end-of-line or EOF */
+		while (token != GUC_EOL && token != 0)
+			token = yylex();
+		/* break out of loop on EOF */
+		if (token == 0)
+			break;
+	}
+
+cleanup:
+	yy_delete_buffer(lex_buffer);
+	/* Each recursion level must save and restore these static variables. */
+	ConfigFileLineno = save_ConfigFileLineno;
+	GUC_flex_fatal_jmp = save_GUC_flex_fatal_jmp;
+	return OK;
+}
+
+/*
+ * Read and parse all config files in a subdirectory in alphabetical order
+ *
+ * includedir is the absolute or relative path to the subdirectory to scan.
+ *
+ * calling_file/calling_lineno identify the source of the request.
+ * Pass NULL/0 if not recursing from an inclusion request.
+ *
+ * See ParseConfigFp for further details.
+ */
+bool
+ParseConfigDirectory(const char *includedir,
+					 const char *calling_file, int calling_lineno,
+					 int depth, int elevel,
+					 ConfigVariable **head_p,
+					 ConfigVariable **tail_p)
+{
+	char	   *directory;
+	DIR		   *d;
+	struct dirent *de;
+	char	  **filenames;
+	int			num_filenames;
+	int			size_filenames;
+	bool		status;
+
+	directory = AbsoluteConfigLocation(includedir, calling_file);
+	d = opendir(directory);
+	if (d == NULL)
+	{
+		pg_log(PG_WARNING, "could not open configuration directory \"%s\"\n",
+			directory);
+		record_config_file_error(psprintf("could not open directory \"%s\"",
+										  directory),
+								 calling_file, calling_lineno,
+								 head_p, tail_p);
+		status = false;
+		goto cleanup;
+	}
+
+	/*
+	 * Read the directory and put the filenames in an array, so we can sort
+	 * them prior to processing the contents.
+	 */
+	size_filenames = 32;
+	filenames = (char **) palloc(size_filenames * sizeof(char *));
+	num_filenames = 0;
+
+	while ((de = readdir(d)) != NULL)
+	{
+		struct stat st;
+		char		filename[MAXPGPATH];
+
+		/*
+		 * Only parse files with names ending in ".conf".  Explicitly reject
+		 * files starting with ".".  This excludes things like "." and "..",
+		 * as well as typical hidden files, backup files, and editor debris.
+		 */
+		if (strlen(de->d_name) < 6)
+			continue;
+		if (de->d_name[0] == '.')
+			continue;
+		if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0)
+			continue;
+
+		join_path_components(filename, directory, de->d_name);
+		canonicalize_path(filename);
+		if (stat(filename, &st) == 0)
+		{
+			if (!S_ISDIR(st.st_mode))
+			{
+				/* Add file to array, increasing its size in blocks of 32 */
+				if (num_filenames >= size_filenames)
+				{
+					size_filenames += 32;
+					filenames = (char **) repalloc(filenames,
+											size_filenames * sizeof(char *));
+				}
+				filenames[num_filenames] = pstrdup(filename);
+				num_filenames++;
+			}
+		}
+		else
+		{
+			/*
+			 * stat does not care about permissions, so the most likely reason
+			 * a file can't be accessed now is if it was removed between the
+			 * directory listing and now.
+			 */
+			pg_log(PG_WARNING, "could not stat file \"%s\"\n", filename);
+			record_config_file_error(psprintf("could not stat file \"%s\"",
+											  filename),
+									 calling_file, calling_lineno,
+									 head_p, tail_p);
+			status = false;
+			goto cleanup;
+		}
+	}
+
+	if (num_filenames > 0)
+	{
+		int			i;
+
+		qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp);
+		for (i = 0; i < num_filenames; i++)
+		{
+			if (!ParseConfigFile(filenames[i], true,
+								 calling_file, calling_lineno,
+								 depth, elevel,
+								 head_p, tail_p))
+			{
+				status = false;
+				goto cleanup;
+			}
+		}
+	}
+	status = true;
+
+cleanup:
+	if (d)
+		closedir(d);
+	pfree(directory);
+	return status;
+}
+
+/*
+ * Free a list of ConfigVariables, including the names and the values
+ */
+void
+FreeConfigVariables(ConfigVariable *list)
+{
+	ConfigVariable *item;
+
+	item = list;
+	while (item)
+	{
+		ConfigVariable *next = item->next;
+
+		FreeConfigVariable(item);
+		item = next;
+	}
+}
+
+/*
+ * Free a single ConfigVariable
+ */
+static void
+FreeConfigVariable(ConfigVariable *item)
+{
+	if (item->name)
+		pfree(item->name);
+	if (item->value)
+		pfree(item->value);
+	if (item->errmsg)
+		pfree(item->errmsg);
+	if (item->filename)
+		pfree(item->filename);
+	pfree(item);
+}
+
+
+/*
+ *		scanstr
+ *
+ * Strip the quotes surrounding the given string, and collapse any embedded
+ * '' sequences and backslash escapes.
+ *
+ * the string returned is palloc'd and should eventually be pfree'd by the
+ * caller.
+ */
+static char *
+GUC_scanstr(const char *s)
+{
+	char	   *newStr;
+	int			len,
+				i,
+				j;
+
+	Assert(s != NULL && s[0] == '\'');
+	len = strlen(s);
+	Assert(len >= 2);
+	Assert(s[len - 1] == '\'');
+
+	/* Skip the leading quote; we'll handle the trailing quote below */
+	s++, len--;
+
+	/* Since len still includes trailing quote, this is enough space */
+	newStr = palloc(len);
+
+	for (i = 0, j = 0; i < len; i++)
+	{
+		if (s[i] == '\\')
+		{
+			i++;
+			switch (s[i])
+			{
+				case 'b':
+					newStr[j] = '\b';
+					break;
+				case 'f':
+					newStr[j] = '\f';
+					break;
+				case 'n':
+					newStr[j] = '\n';
+					break;
+				case 'r':
+					newStr[j] = '\r';
+					break;
+				case 't':
+					newStr[j] = '\t';
+					break;
+				case '0':
+				case '1':
+				case '2':
+				case '3':
+				case '4':
+				case '5':
+				case '6':
+				case '7':
+					{
+						int			k;
+						long		octVal = 0;
+
+						for (k = 0;
+							 s[i + k] >= '0' && s[i + k] <= '7' && k < 3;
+							 k++)
+							octVal = (octVal << 3) + (s[i + k] - '0');
+						i += k - 1;
+						newStr[j] = ((char) octVal);
+					}
+					break;
+				default:
+					newStr[j] = s[i];
+					break;
+			}					/* switch */
+		}
+		else if (s[i] == '\'' && s[i + 1] == '\'')
+		{
+			/* doubled quote becomes just one quote */
+			newStr[j] = s[++i];
+		}
+		else
+			newStr[j] = s[i];
+		j++;
+	}
+
+	/* We copied the ending quote to newStr, so replace with \0 */
+	Assert(j > 0 && j <= len);
+	newStr[--j] = '\0';
+
+	return newStr;
+}