Subject: [PATCH v16 5/5] introduce restore_library

From: Nathan Bossart <[email-masked]>
Date: Wed, 15 Feb 2023 22:06:04 -0800

---
 contrib/basic_archive/Makefile                |   1 +
 contrib/basic_archive/basic_archive.c         | 153 +++++++-
 contrib/basic_archive/meson.build             |   5 +
 contrib/basic_archive/t/001_restore.pl        |  44 +++
 doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
 doc/src/sgml/backup.sgml                      |  40 +-
 doc/src/sgml/basic-archive.sgml               |  31 +-
 doc/src/sgml/config.sgml                      |  53 ++-
 doc/src/sgml/high-availability.sgml           |  18 +-
 src/backend/access/transam/xlog.c             |  20 +-
 src/backend/access/transam/xlogarchive.c      |  96 ++++-
 src/backend/access/transam/xlogrecovery.c     |  14 +-
 src/backend/postmaster/checkpointer.c         |  54 +++
 src/backend/postmaster/startup.c              |  44 ++-
 src/backend/restore/shell_restore.c           |  85 ++++-
 src/backend/utils/misc/guc_tables.c           |  11 +
 src/backend/utils/misc/postgresql.conf.sample |   1 +
 src/include/restore/restore_module.h          | 143 +++++++
 src/include/restore/shell_restore.h           |  16 +-
 src/tools/pgindent/typedefs.list              |   2 +
 20 files changed, 1103 insertions(+), 84 deletions(-)
 create mode 100644 contrib/basic_archive/t/001_restore.pl
 create mode 100644 src/include/restore/restore_module.h

diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..d12d45e0d2 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -5,6 +5,7 @@ PGFILEDESC = "basic_archive - basic archive module"
 
 REGRESS = basic_archive
 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+TAP_TESTS = 1
 # Disabled because these tests require "shared_preload_libraries=basic_archive",
 # which typical installcheck users do not have (e.g. buildfarm clients).
 NO_INSTALLCHECK = 1
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 4d78c31859..862689f981 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
  * a file is successfully archived and then the system crashes before
  * a durable record of the success has been made.
  *
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ *		cp /path/to/archivedir/%f %p
+ *
  * Copyright (c) 2022-2023, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
@@ -33,6 +38,7 @@
 #include "archive/archive_module.h"
 #include "common/int.h"
 #include "miscadmin.h"
+#include "restore/restore_module.h"
 #include "storage/copydir.h"
 #include "storage/fd.h"
 #include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
 static bool check_archive_directory(char **newval, void **extra, GucSource source);
 static bool compare_files(const char *file1, const char *file2);
 static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+									  const char *file, const char *path,
+									  const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+										   const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+												  const char *file,
+												  const char *path);
 
 static const ArchiveModuleCallbacks basic_archive_callbacks = {
 	.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
 	.shutdown_cb = basic_archive_shutdown
 };
 
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+	.startup_cb = NULL,
+	.restore_wal_segment_configured_cb = basic_restore_configured,
+	.restore_wal_segment_cb = basic_restore_wal_segment,
+	.restore_timeline_history_configured_cb = basic_restore_configured,
+	.restore_timeline_history_cb = basic_restore_timeline_history,
+	.timeline_history_exists_configured_cb = basic_restore_configured,
+	.timeline_history_exists_cb = basic_restore_timeline_history_exists,
+	.archive_cleanup_configured_cb = NULL,
+	.archive_cleanup_cb = NULL,
+	.recovery_end_configured_cb = NULL,
+	.recovery_end_cb = NULL,
+	.shutdown_cb = NULL
+};
+
 /*
  * _PG_init
  *
@@ -71,7 +102,7 @@ void
 _PG_init(void)
 {
 	DefineCustomStringVariable("basic_archive.archive_directory",
-							   gettext_noop("Archive file destination directory."),
+							   gettext_noop("Archive file source/destination directory."),
 							   NULL,
 							   &archive_directory,
 							   "",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
 	return &basic_archive_callbacks;
 }
 
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+	return &basic_restore_callbacks;
+}
+
 /*
  * basic_archive_startup
  *
@@ -426,3 +468,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
 	pfree(data);
 	state->private_data = NULL;
 }
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+	return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+						  const char *file, const char *path,
+						  const char *lastRestartPointFileName)
+{
+	return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+							   const char *file, const char *path)
+{
+	return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+	char		archive_path[MAXPGPATH];
+	struct stat st;
+
+	ereport(DEBUG1,
+			(errmsg("restoring \"%s\" via basic_archive",
+					file)));
+
+	/*
+	 * If the file doesn't exist, return false to indicate that there are no
+	 * more files to restore.
+	 */
+	snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+	if (stat(archive_path, &st) != 0)
+	{
+		int			elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not stat file \"%s\": %m",
+						archive_path)));
+		return false;
+	}
+
+	copy_file(archive_path, path);
+
+	ereport(DEBUG1,
+			(errmsg("restored \"%s\" via basic_archive",
+					file)));
+	return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+									  const char *file, const char *path)
+{
+	char		archive_path[MAXPGPATH];
+	struct stat st;
+
+	ereport(DEBUG1,
+			(errmsg("checking existence of \"%s\" via basic_archive",
+					file)));
+
+	snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+	if (stat(archive_path, &st) != 0)
+	{
+		int			elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not stat file \"%s\": %m",
+						archive_path)));
+		return false;
+	}
+
+	ereport(DEBUG1,
+			(errmsg("verified existence of \"%s\" via basic_archive",
+					file)));
+	return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index bc1380e6f6..4e9f5002de 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
     # which typical runningcheck users do not have (e.g. buildfarm clients).
     'runningcheck': false,
   },
+  'tap': {
+    'tests': [
+      't/001_restore.pl',
+    ],
+  },
 }
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..f7fa124695
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index 7ed50a3b52..bf2fdb313a 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
 
  <para>
   PostgreSQL provides infrastructure to create custom modules for continuous
-  archiving (see <xref linkend="continuous-archiving"/>).  While archiving via
-  a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
-  simpler, a custom archive module will often be considerably more robust and
-  performant.
+  archiving and recovery (see <xref linkend="continuous-archiving"/>).  While a
+  shell command (i.e., <xref linkend="guc-archive-command"/>,
+  <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+  often be considerably more robust and performant.
  </para>
 
  <para>
   The <filename>contrib/basic_archive</filename> module contains a working
-  example, which demonstrates some useful techniques.
+  example, which demonstrates some useful techniques.  As demonstrated in
+  <filename>basic_archive</filename> a single module may serve as both an
+  archive module and a restore module.
  </para>
 
  <sect1 id="archive-modules">
@@ -166,4 +168,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
    </sect3>
   </sect2>
  </sect1>
+
+ <sect1 id="restore-modules">
+  <title>Restore Modules</title>
+  <indexterm zone="restore-modules">
+   <primary>Restore Modules</primary>
+  </indexterm>
+
+  <para>
+   When a custom <xref linkend="guc-restore-library"/> is configured,
+   PostgreSQL will use the module for restore actions.  It is
+   ultimately up to the module to decide how to accomplish each task,
+   but some recommendations are listed at
+   <xref linkend="backup-pitr-recovery"/>.
+  </para>
+
+  <para>
+   Restore modules must at least consist of an initialization function
+   (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+   <xref linkend="restore-module-callbacks"/>).  However, restore modules are
+   also permitted to do much more (e.g., declare GUCs and register background
+   workers).
+  </para>
+
+  <sect2 id="restore-module-init">
+   <title>Initialization Functions</title>
+   <indexterm zone="restore-module-init">
+    <primary>_PG_restore_module_init</primary>
+   </indexterm>
+
+   <para>
+    An restore library is loaded by dynamically loading a shared library with
+    the <xref linkend="guc-restore-library"/>'s name as the library base name.
+    The normal library search path is used to locate the library.  To provide
+    the required restore module callbacks and to indicate that the library is
+    actually a restore module, it needs to provide a function named
+    <function>_PG_restore_module_init</function>.  The result of the function
+    must be a pointer to a struct of type
+    <structname>RestoreModuleCallbacks</structname>, which contains everything
+    that the core code needs to know how to make use of the restore module.
+    The return value needs to be of server lifetime, which is typically
+    achieved by defining it as a <literal>static const</literal> variable in
+    global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+    RestoreStartupCB startup_cb;
+    RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+    RestoreWalSegmentCB restore_wal_segment_cb;
+    RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+    RestoreTimelineHistoryCB restore_timeline_history_cb;
+    TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+    TimelineHistoryExistsCB timeline_history_exists_cb;
+    ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+    ArchiveCleanupCB archive_cleanup_cb;
+    RecoveryEndConfiguredCB recovery_end_configured_cb;
+    RecoveryEndCB recovery_end_cb;
+    RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+    The <function>restore_wal_segment_configured_cb</function>,
+    <function>restore_wal_segment_cb</function>,
+    <function>restore_timeline_history_configured_cb</function>,
+    <function>restore_timeline_history_cb</function>,
+    <function>timeline_history_exists_configured_cb</function>, and
+    <function>timeline_history_exists_cb</function> callbacks are required for
+    archive recovery but optional for streaming replication.  The others are
+    always optional.
+   </para>
+  </sect2>
+
+  <sect2 id="restore-module-callbacks">
+   <title>Restore Module Callbacks</title>
+
+   <para>
+    The restore callbacks define the actual behavior of the module.  The server
+    will call them as required to execute restore actions.
+   </para>
+
+   <sect3 id="restore-module-startup">
+    <title>Startup Callback</title>
+
+    <para>
+     The <function>startup_cb</function> callback is called shortly after the
+     module is loaded.  This callback can be used to perform any additional
+     initialization required.  If the restore module has state, it can use
+     <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-restore-wal-segment-configured">
+    <title>Restore WAL Segment Configured Callback</title>
+    <para>
+     The <function>restore_wal_segment_configured_cb</function> callback is
+     called to determine whether the module is fully configured and ready to
+     accept requests to retrieve WAL files (e.g., its configuration parameters
+     are set to valid values).  If no
+     <function>restore_wal_segment_configured_cb</function> is defined, the
+     server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB)  (RestoreModuleState *state);
+</programlisting>
+
+     If <literal>true</literal> is returned, the server will retrieve WAL files
+     by calling the <function>restore_wal_segment_cb</function> callback.  If
+     <literal>false</literal> is returned, the server will not use the
+     <function>restore_wal_segment_cb</function> callback to retrieve WAL
+     files.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-restore-wal-segment">
+    <title>Restore WAL Segment Callback</title>
+    <para>
+     The <function>restore_wal_segment_cb</function> callback is called to
+     retrieve a single archived segment of the WAL file series for archive
+     recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+     This callback must return <literal>true</literal> only if the file was
+     successfully retrieved.  If the file is not available in the archives, the
+     callback must return <literal>false</literal>.
+     <replaceable>file</replaceable> will contain just the file name
+     of the WAL file to retrieve, while <replaceable>path</replaceable>
+     contains the destination's relative path (including the file name).
+     <replaceable>lastRestartPointFileName</replaceable> will contain the name
+     of the file containing the last valid restart point.  That is the earliest
+     file that must be kept to allow a restore to be restartable, so this
+     information can be used to truncate the archive to just the minimum
+     required to support restarting from the current restore.
+     <replaceable>lastRestartPointFileName</replaceable> is typically only used
+     by warm-standby configurations (see <xref linkend="warm-standby"/>).  Note
+     that if multiple standby servers are restoring from the same archive
+     directory, you will need to ensure that you do not delete WAL files until
+     they are no longer needed by any of the servers.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-restore-timeline-history-configured">
+    <title>Restore Timeline History Configured Callback</title>
+    <para>
+     The <function>restore_timeline_history_configured_cb</function> callback
+     is called to determine whether the module is fully configured and ready to
+     accept requests to retrieve timeline history files (e.g., its
+     configuration parameters are set to valid values).  If no
+     <function>restore_timeline_history_configured_cb</function> is defined,
+     the server always assumes the module is not configured to retrieve
+     timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB)  (RestoreModuleState *state);
+</programlisting>
+
+     If <literal>true</literal> is returned, the server will retrieve timeline
+     history files by calling the
+     <function>restore_timeline_history_cb</function> callback.  If
+     <literal>false</literal> is returned, the server will not use the
+     <function>restore_timeline_history_cb</function> callback to retrieve
+     timeline history files.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-restore-timeline-history">
+    <title>Restore Timeline History Callback</title>
+    <para>
+     The <function>restore_timeline_history_cb</function> callback is called to
+     retrieve a single archived timeline history file for archive recovery or
+     streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+     This callback must return <literal>true</literal> only if the file was
+     successfully retrieved.  If the file is not available in the archives, the
+     callback must return <literal>false</literal>.
+     <replaceable>file</replaceable> will contain just the file name
+     of the WAL file to retrieve, while <replaceable>path</replaceable>
+     contains the destination's relative path (including the file name).
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-timeline-history-exists-configured">
+    <title>Timeline History Exists Configured Callback</title>
+    <para>
+     The <function>timeline_history_exists_configured_cb</function> callback is
+     called to determine whether the module is fully configured and ready to
+     accept requests to determine whether a timeline history file exists in the
+     archives (e.g., its configuration parameters are set to valid values).  If
+     no <function>timeline_history_exists_configured_cb</function> is defined,
+     the server always assumes the module is not configured to check whether
+     certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB)  (RestoreModuleState *state);
+</programlisting>
+
+     If <literal>true</literal> is returned, the server will check whether
+     certain timeline history files are present in the archives by calling the
+     <function>timeline_history_exists_cb</function> callback.  If
+     <literal>false</literal> is returned, the server will not use the
+     <function>timeline_history_exists_cb</function> callback to check if
+     timeline history files exist in the archives.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-timeline-history-exists">
+    <title>Timeline History Exists Callback</title>
+    <para>
+     The <function>timeline_history_exists_cb</function> callback is called to
+     check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+     This callback must return <literal>true</literal> only if the file is
+     present in the archives.  If the file is not available in the archives,
+     the callback must return <literal>false</literal>.
+     <replaceable>file</replaceable> will contain just the file name
+     of the timeline history file.
+    </para>
+
+    <para>
+     Some restore modules might not have a dedicated way to determine whether a
+     timeline history file exists.  For example,
+     <varname>restore_command</varname> only tells the server how to retrieve
+     files.  In this case, the <function>timeline_history_exists_cb</function>
+     callback should copy the file to <replaceable>path</replaceable>, which
+     contains the destination's relative path (including the file name), and
+     the restore module should set
+     <structfield>state->timeline_history_exists_cb_copies</structfield> to
+     <literal>true</literal>.  It is recommended to set this flag in the
+     module's <function>startup_cb</function> callback.  This flag tells the
+     server that it should verify that the file was successfully retrieved
+     after invoking this callback.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-archive-cleanup-configured">
+    <title>Archive Cleanup Configured Callback</title>
+    <para>
+     The <function>archive_cleanup_configured_cb</function> callback is called
+     to determine whether the module is fully configured and ready to accept
+     requests to remove old WAL files from the archives (e.g., its
+     configuration parameters are set to valid values).  If no
+     <function>archive_cleanup_configured_cb</function> is defined, the server
+     always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB)  (RestoreModuleState *state);
+</programlisting>
+
+     If <literal>true</literal> is returned, the server will remove old WAL
+     files by calling the <function>archive_cleanup_cb</function> callback.  If
+     <literal>false</literal> is returned, the server will not use the
+     <function>archive_cleanup_cb</function> callback to remove old WAL files.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-archive-cleanup">
+    <title>Archive Cleanup Callback</title>
+    <para>
+     The <function>archive_cleanup_cb</function> callback is called at every
+     restart point by the checkpointer process and is intended to provide a
+     mechanism for cleaning up old archived WAL files that are no longer
+     needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+     <replaceable>lastRestartPointFileName</replaceable> will contain the
+     name of the file that includes the last valid restart point, like in
+     <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-recovery-end-configured">
+    <title>Recovery End Configured Callback</title>
+    <para>
+     The <function>recovery_end_configured_cb</function> callback is called to
+     determine whether the module is fully configured and ready to execute its
+     <function>recovery_end_cb</function> callback once at the end of recovery
+     (e.g., its configuration parameters are set to valid values).  If no
+     <function>recovery_end_configured_cb</function> is defined, the server
+     always assumes the module is not configured to execute its
+     <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB)  (RestoreModuleState *state);
+</programlisting>
+
+     If <literal>true</literal> is returned, the server will execute the
+     <function>recovery_end_cb</function> callback once at the end of recovery.
+     If <literal>false</literal> is returned, the server will not use the
+     <function>recovery_end_cb</function> callback at the end of recovery.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-recovery-end">
+    <title>Recovery End Callback</title>
+    <para>
+     The <function>recovery_end_cb</function> callback is called once at the
+     end of recovery and is intended to provide a mechanism for cleanup
+     following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+     <replaceable>lastRestartPointFileName</replaceable> will contain the name
+     of the file containing the last valid restart point, like in
+     <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+    </para>
+   </sect3>
+
+   <sect3 id="restore-module-shutdown">
+    <title>Shutdown Callback</title>
+    <para>
+     The <function>shutdown_cb</function> callback is called when a process
+     that has loaded the restore module exits (e.g., after an error) or the
+     value of <xref linkend="guc-restore-library"/> changes.  If no
+     <function>shutdown_cb</function> is defined, no special action is taken in
+     these situations.  If the restore module has state, this callback should
+     free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+    </para>
+   </sect3>
+  </sect2>
+ </sect1>
 </chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 8cb24d6ae5..4b20651234 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
    <para>
     The key part of all this is to set up a recovery configuration that
     describes how you want to recover and how far the recovery should
-    run.  The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+    run.  The one thing that you absolutely must specify is either
+    <varname>restore_command</varname> or a <varname>restore_library</varname>,
     which tells <productname>PostgreSQL</productname> how to retrieve archived
-    WAL file segments.  Like the <varname>archive_command</varname>, this is
+    WAL file segments.
+   </para>
+
+   <para>
+    Like the <varname>archive_library</varname> parameter,
+    <varname>restore_library</varname> is a shared library.  Since such
+    libraries are written in <literal>C</literal>, creating your own may
+    require considerably more effort than writing a shell command.  However,
+    restore modules can be more performance than restoring via shell, and they
+    will have access to many useful server resources.  For more information
+    about creating a <varname>restore_library</varname>, see
+    <xref linkend="restore-modules"/>.
+   </para>
+
+   <para>
+    Like the <varname>archive_command</varname>,
+    <varname>restore_command</varname> is
     a shell command string.  It can contain <literal>%f</literal>, which is
     replaced by the name of the desired WAL file, and <literal>%p</literal>,
     which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1218,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
    </para>
 
    <para>
-    It is important that the command return nonzero exit status on failure.
-    The command <emphasis>will</emphasis> be called requesting files that are not
-    present in the archive; it must return nonzero when so asked.  This is not
-    an error condition.  An exception is that if the command was terminated by
+    It is important that the <varname>restore_command</varname> return nonzero
+    exit status on failure, or, if you are using a
+    <varname>restore_library</varname>, that the restore callbacks return
+    <literal>false</literal> on failure.  The command or library
+    <emphasis>will</emphasis> be called requesting files that are not
+    present in the archive; it must fail when so asked.  This is not
+    an error condition.  An exception is that if the
+    <varname>restore_command</varname> was terminated by
     a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
     part of a database server shutdown) or an error by the shell (such as
     command not found), then recovery will abort and the server will not start
-    up.
+    up.  Likewise, if the restore callbacks provided by the
+    <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+    <literal>FATAL</literal>, recovery will abort and the server won't start.
    </para>
 
    <para>
@@ -1232,7 +1255,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
     close as possible given the available WAL segments).  Therefore, a normal
     recovery will end with a <quote>file not found</quote> message, the exact text
     of the error message depending upon your choice of
-    <varname>restore_command</varname>.  You may also see an error message
+    <varname>restore_command</varname> or <varname>restore_library</varname>.
+    You may also see an error message
     at the start of recovery for a file named something like
     <filename>00000001.history</filename>.  This is also normal and does not
     indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
  </indexterm>
 
  <para>
-  <filename>basic_archive</filename> is an example of an archive module.  This
-  module copies completed WAL segment files to the specified directory.  This
-  may not be especially useful, but it can serve as a starting point for
-  developing your own archive module.  For more information about archive
-  modules, see <xref linkend="archive-modules"/>.
+  <filename>basic_archive</filename> is an example of an archive and restore
+  module.  This module copies completed WAL segment files to or from the
+  specified directory.  This may not be especially useful, but it can serve as
+  a starting point for developing your own archive and restore modules.  For
+  more information about archive and restore modules, see\
+  <xref linkend="archive-and-restore-modules"/>.
  </para>
 
  <para>
-  In order to function, this module must be loaded via
+  For use as an archive module, this module must be loaded via
   <xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
-  must be enabled.
+  must be enabled.  For use as a restore module, this module must be loaded
+  via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+  <xref linkend="runtime-config-wal-archive-recovery"/>).
  </para>
 
  <sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
     </term>
     <listitem>
      <para>
-      The directory where the server should copy WAL segment files.  This
-      directory must already exist.  The default is an empty string, which
-      effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
-      is enabled, the server will accumulate WAL segment files in the
-      expectation that a value will soon be provided.
+      The directory where the server should copy WAL segment files to or from.
+      This directory must already exist.  The default is an empty string,
+      which, when used for archiving, effectively halts WAL archival, but if
+      <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+      WAL segment files in the expectation that a value will soon be provided.
+      When an empty string is used for recovery, restore will fail.
      </para>
     </listitem>
    </varlistentry>
@@ -46,7 +50,7 @@
 
   <para>
    These parameters must be set in <filename>postgresql.conf</filename>.
-   Typical usage might be:
+   Typical usage as an archive module might be:
   </para>
 
 <programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
   <title>Notes</title>
 
   <para>
+   When <filename>basic_archive</filename> is used as an archive module,
    Server crashes may leave temporary files with the prefix
    <filename>archtemp</filename> in the archive directory.  It is recommended to
    delete such files before restarting the server after a crash.  It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 3839c72c86..cdc2f902e0 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3751,7 +3751,8 @@ include_dir 'conf.d'
      recovery when the end of archived WAL is reached, but will keep trying to
      continue recovery by connecting to the sending server as specified by the
      <varname>primary_conninfo</varname> setting and/or by fetching new WAL
-     segments using <varname>restore_command</varname>.  For this mode, the
+     segments using <varname>restore_command</varname> or
+     <varname>restore_library</varname>.  For this mode, the
      parameters from this section and <xref
      linkend="runtime-config-replication-standby"/> are of interest.
      Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3779,7 +3780,8 @@ include_dir 'conf.d'
       <listitem>
        <para>
         The local shell command to execute to retrieve an archived segment of
-        the WAL file series. This parameter is required for archive recovery,
+        the WAL file series.  Either <varname>restore_command</varname> or
+        <xref linkend="guc-restore-library"/> is required for archive recovery,
         but optional for streaming replication.
         Any <literal>%f</literal> in the string is
         replaced by the name of the file to retrieve from the archive,
@@ -3814,7 +3816,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
 
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
-        file or on the server command line.
+        file or on the server command line.  It is only used if
+        <varname>restore_library</varname> is set to an empty string.  If both
+        <varname>restore_command</varname> and
+        <varname>restore_library</varname> are set, an error will be raised.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="guc-restore-library" xreflabel="restore_library">
+      <term><varname>restore_library</varname> (<type>string</type>)
+      <indexterm>
+       <primary><varname>restore_library</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        The library to use for restore actions, including retrieving archived
+        files and executing tasks at restartpoints and at recovery end.  Either
+        <xref linkend="guc-restore-command"/> or
+        <varname>restore_library</varname> is required for archive recovery,
+        but optional for streaming replication.  If this parameter is set to an
+        empty string (the default), restoring via shell is enabled, and
+        <varname>restore_command</varname>,
+        <varname>archive_cleanup_command</varname>, and
+        <varname>recovery_end_command</varname> are used.  If both
+        <varname>restore_library</varname> and any of
+        <varname>restore_command</varname>,
+        <varname>archive_cleanup_command</varname>, or
+        <varname>recovery_end_command</varname> are set, an error will be
+        raised.  Otherwise, the specified shared library is used for restoring.
+        For more information, see <xref linkend="restore-modules"/>.
+       </para>
+
+       <para>
+        This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
        </para>
       </listitem>
      </varlistentry>
@@ -3859,7 +3896,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
        </para>
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
-        file or on the server command line.
+        file or on the server command line.  It is only used if
+        <varname>restore_library</varname> is set to an empty string.  If both
+        <varname>archive_cleanup_command</varname> and
+        <varname>restore_library</varname> are set, an error will be raised.
        </para>
       </listitem>
      </varlistentry>
@@ -3888,7 +3928,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
        </para>
        <para>
         This parameter can only be set in the <filename>postgresql.conf</filename>
-        file or on the server command line.
+        file or on the server command line.  It is only used if
+        <varname>restore_library</varname> is set to an empty string.  If both
+        <varname>recovery_end_command</varname> and
+        <varname>restore_library</varname> are set, an error will be raised.
        </para>
       </listitem>
      </varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 5f9257313a..09705a14a2 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
    <para>
     In standby mode, the server continuously applies WAL received from the
     primary server. The standby server can read WAL from a WAL archive
-    (see <xref linkend="guc-restore-command"/>) or directly from the primary
+    (see <xref linkend="guc-restore-command"/> and
+    <xref linkend="guc-restore-library"/>) or directly from the primary
     over a TCP connection (streaming replication). The standby server will
     also attempt to restore any WAL found in the standby cluster's
     <filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
 
    <para>
     At startup, the standby begins by restoring all WAL available in the
-    archive location, calling <varname>restore_command</varname>. Once it
+    archive location, either by calling <varname>restore_command</varname> or
+    by executing the <varname>restore_library</varname>'s callbacks.  Once it
     reaches the end of WAL available there and <varname>restore_command</varname>
+    or <varname>restore_library</varname>
     fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
     If that fails, and streaming replication has been configured, the
     standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
     server (see <xref linkend="backup-pitr-recovery"/>). Create a file
     <link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
     in the standby's cluster data
-    directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+    directory. Set <xref linkend="guc-restore-command"/> or
+    <xref linkend="guc-restore-library"/> to copy files from
     the WAL archive. If you plan to have multiple standby servers for high
     availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
     <literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
 
    <note>
      <para>
-     <xref linkend="guc-restore-command"/> should return immediately
+     <xref linkend="guc-restore-command"/> and
+     <xref linkend="guc-restore-library"/> should return immediately
      if the file does not exist; the server will retry the command again if
      necessary.
     </para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
 
    <para>
     If you're using a WAL archive, its size can be minimized using the <xref
-    linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
-    longer required by the standby server.
+    linkend="guc-archive-cleanup-command"/> parameter or the
+    <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+    files that are no longer required by the standby server.
     The <application>pg_archivecleanup</application> utility is designed specifically to
     be used with <varname>archive_cleanup_command</varname> in typical single-standby
     configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1307748b4e..c79547e403 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,7 +84,7 @@
 #include "replication/snapbuild.h"
 #include "replication/walreceiver.h"
 #include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
 #include "storage/bufmgr.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
@@ -4983,18 +4983,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
 							TimeLineID newTLI)
 {
 	/*
-	 * Execute the recovery_end_command, if any.
+	 * Execute the recovery-end callback, if any.
 	 *
-	 * The command is provided the archive file cutoff point for use during
+	 * The callback is provided the archive file cutoff point for use during
 	 * log shipping replication.  All files earlier than this point can be
 	 * deleted from the archive, though there is no requirement to do so.
 	 */
-	if (shell_recovery_end_configured())
+	if (recovery_end_configured())
 	{
 		char		lastRestartPointFname[MAXFNAMELEN];
 
 		GetOldestRestartPointFileName(lastRestartPointFname);
-		shell_recovery_end(lastRestartPointFname);
+		RestoreCallbacks->recovery_end_cb(restore_module_state,
+										  lastRestartPointFname);
 	}
 
 	/*
@@ -7453,18 +7454,19 @@ CreateRestartPoint(int flags)
 							   timestamptz_to_str(xtime)) : 0));
 
 	/*
-	 * Finally, execute archive_cleanup_command, if any.
+	 * Finally, execute archive-cleanup callback, if any.
 	 *
-	 * The command is provided the archive file cutoff point for use during
+	 * The callback is provided the archive file cutoff point for use during
 	 * log shipping replication.  All files earlier than this point can be
 	 * deleted from the archive, though there is no requirement to do so.
 	 */
-	if (shell_archive_cleanup_configured())
+	if (archive_cleanup_configured())
 	{
 		char		lastRestartPointFname[MAXFNAMELEN];
 
 		GetOldestRestartPointFileName(lastRestartPointFname);
-		shell_archive_cleanup(lastRestartPointFname);
+		RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+											 lastRestartPointFname);
 	}
 
 	return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index a925ac22f6..316b866af0 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,15 +23,22 @@
 #include "access/xlog_internal.h"
 #include "access/xlogarchive.h"
 #include "common/archive.h"
+#include "fmgr.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/startup.h"
 #include "postmaster/pgarch.h"
 #include "replication/walsender.h"
+#include "restore/restore_module.h"
 #include "restore/shell_restore.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/lwlock.h"
+#include "utils/memutils.h"
+
+char	   *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
 
 /*
  * Attempt to retrieve the specified file from off-line archival storage.
@@ -75,15 +82,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 	switch (archive_type)
 	{
 		case ARCHIVE_TYPE_WAL_SEGMENT:
-			if (!shell_restore_file_configured())
+			if (!restore_wal_segment_configured())
 				goto not_available;
 			break;
 		case ARCHIVE_TYPE_TIMELINE_HISTORY:
-			if (!shell_restore_file_configured())
+			if (!restore_timeline_history_configured())
 				goto not_available;
 			break;
 		case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
-			if (!shell_restore_file_configured())
+			if (!timeline_history_exists_configured())
 				goto not_available;
 			break;
 	}
@@ -175,14 +182,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 	switch (archive_type)
 	{
 		case ARCHIVE_TYPE_WAL_SEGMENT:
-			ret = shell_restore_wal_segment(xlogfname, xlogpath,
-											lastRestartPointFname);
+			ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+														   xlogfname, xlogpath,
+														   lastRestartPointFname);
 			break;
 		case ARCHIVE_TYPE_TIMELINE_HISTORY:
-			ret = shell_restore_timeline_history(xlogfname, xlogpath);
+			ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+																xlogfname, xlogpath);
 			break;
 		case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
-			ret = shell_restore_timeline_history(xlogfname, xlogpath);
+			ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+															   xlogfname, xlogpath);
 			break;
 	}
 
@@ -190,6 +200,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 
 	if (ret)
 	{
+		/*
+		 * Some restore functions might not copy the file (e.g., checking
+		 * whether a timeline history file exists), but they can set a flag to
+		 * tell us if they do.  We only need to verify file existence if this
+		 * flag is enabled.
+		 */
+		if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+			!restore_module_state->timeline_history_exists_cb_copies)
+			return true;
+
 		/*
 		 * command apparently succeeded, but let's make sure the file is
 		 * really there now and has the correct size.
@@ -631,3 +651,65 @@ XLogArchiveCleanup(const char *xlog)
 	unlink(archiveStatusPath);
 	/* should we complain about failure? */
 }
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks.  The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+	RestoreModuleInit init;
+
+	/*
+	 * If the shell command is enabled, use our special initialization
+	 * function.  Otherwise, load the library and call its
+	 * _PG_restore_module_init().
+	 */
+	if (restoreLibrary[0] == '\0')
+		init = shell_restore_init;
+	else
+		init = (RestoreModuleInit)
+			load_external_function(restoreLibrary, "_PG_restore_module_init",
+								   false, NULL);
+
+	if (init == NULL)
+		ereport(ERROR,
+				(errmsg("restore modules have to define the symbol "
+						"_PG_restore_module_init")));
+
+	RestoreCallbacks = (*init) ();
+
+	/* restore state should be freed before calling this function */
+	Assert(restore_module_state == NULL);
+	restore_module_state = (RestoreModuleState *)
+		MemoryContextAllocZero(TopMemoryContext,
+							   sizeof(RestoreModuleState));
+
+	if (RestoreCallbacks->startup_cb != NULL)
+		RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined.  Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+	if (RestoreCallbacks != NULL &&
+		RestoreCallbacks->shutdown_cb != NULL &&
+		restore_module_state != NULL)
+		RestoreCallbacks->shutdown_cb(restore_module_state);
+
+	if (restore_module_state != NULL)
+	{
+		pfree(restore_module_state);
+		restore_module_state = NULL;
+	}
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index d43c229eca..8d231dcc82 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -50,6 +50,7 @@
 #include "postmaster/startup.h"
 #include "replication/slot.h"
 #include "replication/walreceiver.h"
+#include "restore/restore_module.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/latch.h"
@@ -1077,18 +1078,21 @@ validateRecoveryParameters(void)
 	if (StandbyModeRequested)
 	{
 		if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
-			(recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+			(!restore_wal_segment_configured() ||
+			 !restore_timeline_history_configured() ||
+			 !timeline_history_exists_configured()))
 			ereport(WARNING,
-					(errmsg("specified neither primary_conninfo nor restore_command"),
+					(errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
 					 errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
 	}
 	else
 	{
-		if (recoveryRestoreCommand == NULL ||
-			strcmp(recoveryRestoreCommand, "") == 0)
+		if (!restore_wal_segment_configured() ||
+			!restore_timeline_history_configured() ||
+			!timeline_history_exists_configured())
 			ereport(FATAL,
 					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
-					 errmsg("must specify restore_command when standby mode is not enabled")));
+					 errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
 	}
 
 	/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index ace9893d95..4ded7f3ff3 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
 #include "postmaster/bgwriter.h"
 #include "postmaster/interrupt.h"
 #include "replication/syncrep.h"
+#include "restore/restore_module.h"
 #include "storage/bufmgr.h"
 #include "storage/condition_variable.h"
 #include "storage/fd.h"
@@ -233,6 +234,25 @@ CheckpointerMain(void)
 												 ALLOCSET_DEFAULT_SIZES);
 	MemoryContextSwitchTo(checkpointer_context);
 
+	/*
+	 * Load restore_library so that we can use its archive-cleanup callback,
+	 * if one is defined.
+	 *
+	 * We also take this opportunity to set up the before_shmem_exit hook for
+	 * the shutdown callback and to check that only one of restore_library,
+	 * archive_cleanup_command is set.  Note that we emit a WARNING upon
+	 * detecting misconfigured parameters, and we clear the callbacks so that
+	 * the archive-cleanup functionality is skipped.  We judge this
+	 * functionality to not be important enough to require blocking
+	 * checkpoints or shutting down the server when the parameters are
+	 * misconfigured.
+	 */
+	before_shmem_exit(call_restore_module_shutdown_cb, 0);
+	if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+										 archiveCleanupCommand, "archive_cleanup_command",
+										 WARNING))
+		LoadRestoreCallbacks();
+
 	/*
 	 * If an exception is encountered, processing resumes here.
 	 *
@@ -547,6 +567,10 @@ HandleCheckpointerInterrupts(void)
 
 	if (ConfigReloadPending)
 	{
+		bool		archive_cleanup_settings_changed = false;
+		char	   *prevRestoreLibrary = pstrdup(restoreLibrary);
+		char	   *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
 		ConfigReloadPending = false;
 		ProcessConfigFile(PGC_SIGHUP);
 
@@ -562,6 +586,36 @@ HandleCheckpointerInterrupts(void)
 		 * because of SIGHUP.
 		 */
 		UpdateSharedMemoryConfig();
+
+		/*
+		 * If the archive-cleanup settings have changed, call the currently
+		 * loaded shutdown callback and clear all the restore callbacks.
+		 * There's presently no good way to unload a library besides
+		 * restarting the process, and there should be little harm in leaving
+		 * it around, so we just leave it loaded.
+		 */
+		if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+			strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+		{
+			archive_cleanup_settings_changed = true;
+			call_restore_module_shutdown_cb(0, (Datum) 0);
+			RestoreCallbacks = NULL;
+		}
+
+		/*
+		 * As in CheckpointerMain(), we only emit a WARNING if we detect that
+		 * both restore_library and archive_cleanup_command are set.  We do
+		 * this even if the archive-cleanup settings haven't changed to remind
+		 * the user about the misconfiguration.
+		 */
+		if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+											 archiveCleanupCommand, "archive_cleanup_command",
+											 WARNING) &&
+			archive_cleanup_settings_changed)
+			LoadRestoreCallbacks();
+
+		pfree(prevRestoreLibrary);
+		pfree(prevArchiveCleanupCommand);
 	}
 	if (ShutdownRequestPending)
 	{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index adce0ffcef..8ad570c725 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -29,6 +29,7 @@
 #include "pgstat.h"
 #include "postmaster/interrupt.h"
 #include "postmaster/startup.h"
+#include "restore/restore_module.h"
 #include "storage/ipc.h"
 #include "storage/latch.h"
 #include "storage/pmsignal.h"
@@ -148,13 +149,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
  * Re-read the config file.
  *
  * If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it.  Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
  */
 static void
 StartupRereadConfig(void)
 {
 	char	   *conninfo = pstrdup(PrimaryConnInfo);
 	char	   *slotname = pstrdup(PrimarySlotName);
+	char	   *prevRestoreLibrary = pstrdup(restoreLibrary);
+	char	   *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+	char	   *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
 	bool		tempSlot = wal_receiver_create_temp_slot;
 	bool		conninfoChanged;
 	bool		slotnameChanged;
@@ -176,6 +181,30 @@ StartupRereadConfig(void)
 
 	if (conninfoChanged || slotnameChanged || tempSlotChanged)
 		StartupRequestWalReceiverRestart();
+
+	/*
+	 * If the restore settings have changed, call the currently loaded
+	 * shutdown callback and load the new callbacks.  There's presently no
+	 * good way to unload a library besides restarting the process, and there
+	 * should be little harm in leaving it around, so we just leave it loaded.
+	 */
+	(void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+											recoveryRestoreCommand, "restore_command",
+											ERROR);
+	(void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+											recoveryEndCommand, "recovery_end_command",
+											ERROR);
+	if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+		strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+		strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+	{
+		call_restore_module_shutdown_cb(0, (Datum) 0);
+		LoadRestoreCallbacks();
+	}
+
+	pfree(prevRestoreLibrary);
+	pfree(prevRestoreCommand);
+	pfree(prevRecoveryEndCommand);
 }
 
 /* Handle various signals that might be sent to the startup process */
@@ -285,6 +314,19 @@ StartupProcessMain(void)
 	 */
 	sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
 
+	/*
+	 * Check for invalid combinations of the command/library parameters and
+	 * load the callbacks.
+	 */
+	before_shmem_exit(call_restore_module_shutdown_cb, 0);
+	(void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+											recoveryRestoreCommand, "restore_command",
+											ERROR);
+	(void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+											recoveryEndCommand, "recovery_end_command",
+											ERROR);
+	LoadRestoreCallbacks();
+
 	/*
 	 * Do what we came for.
 	 */
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index bc1f4a431a..ce77fde66b 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
  * shell_restore.c
  *
  * These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC).  It is used as the default, but other modules may
+ * define their own restore logic.
  *
  * Copyright (c) 2023, PostgreSQL Global Development Group
  *
@@ -22,25 +23,76 @@
 #include "common/archive.h"
 #include "common/percentrepl.h"
 #include "postmaster/startup.h"
+#include "restore/restore_module.h"
 #include "restore/shell_restore.h"
 #include "storage/ipc.h"
 #include "utils/wait_event.h"
 
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
 static bool shell_restore_file(const char *file, const char *path,
 							   const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+									  const char *file, const char *path,
+									  const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+										   const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+								  const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+							   const char *lastRestartPointFileName);
 static void ExecuteRecoveryCommand(const char *command,
 								   const char *commandName,
 								   bool failOnSignal,
 								   uint32 wait_event_info,
 								   const char *lastRestartPointFileName);
 
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+	.startup_cb = shell_restore_startup,
+	.restore_wal_segment_configured_cb = shell_restore_file_configured,
+	.restore_wal_segment_cb = shell_restore_wal_segment,
+	.restore_timeline_history_configured_cb = shell_restore_file_configured,
+	.restore_timeline_history_cb = shell_restore_timeline_history,
+	.timeline_history_exists_configured_cb = shell_restore_file_configured,
+	.timeline_history_exists_cb = shell_restore_timeline_history,
+	.archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+	.archive_cleanup_cb = shell_archive_cleanup,
+	.recovery_end_configured_cb = shell_recovery_end_configured,
+	.recovery_end_cb = shell_recovery_end,
+	.shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+	return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+	state->timeline_history_exists_cb_copies = true;
+}
+
 /*
  * Attempt to execute a shell-based restore command to retrieve a WAL segment.
  *
  * Returns true if the command has succeeded, false otherwise.
  */
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+						  const char *file, const char *path,
 						  const char *lastRestartPointFileName)
 {
 	return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
  *
  * Returns true if the command has succeeded, false otherwise.
  */
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+							   const char *file, const char *path)
 {
 	char		lastRestartPointFname[MAXPGPATH];
 
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
 /*
  * Check whether restore_command is supplied.
  */
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
 {
 	return recoveryRestoreCommand[0] != '\0';
 }
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
 /*
  * Check whether archive_cleanup_command is supplied.
  */
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
 {
 	return archiveCleanupCommand[0] != '\0';
 }
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
 /*
  * Attempt to execute a shell-based archive cleanup command.
  */
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+					  const char *lastRestartPointFileName)
 {
 	ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
 						   false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
 /*
  * Check whether recovery_end_command is supplied.
  */
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
 {
 	return recoveryEndCommand[0] != '\0';
 }
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
 /*
  * Attempt to execute a shell-based end-of-recovery command.
  */
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+				   const char *lastRestartPointFileName)
 {
 	ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
 						   WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 4c58574166..21e91908aa 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -67,6 +67,7 @@
 #include "replication/logicallauncher.h"
 #include "replication/slot.h"
 #include "replication/syncrep.h"
+#include "restore/restore_module.h"
 #include "storage/bufmgr.h"
 #include "storage/large_object.h"
 #include "storage/pg_shmem.h"
@@ -3828,6 +3829,16 @@ struct config_string ConfigureNamesString[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+			gettext_noop("Sets the library that will be called for restore actions."),
+			NULL
+		},
+		&restoreLibrary,
+		"",
+		NULL, NULL, NULL
+	},
+
 	{
 		{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
 			gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d08d55c3fe..2f9ad7e1bd 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -272,6 +272,7 @@
 				# placeholders: %p = path of file to restore
 				#               %f = file name only
 				# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = ''		# library to use for restore actions
 #archive_cleanup_command = ''	# command to execute at every restartpoint
 #recovery_end_command = ''	# command to execute at completion of recovery
 
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..b9a5810aa8
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ *		Exports for restore modules.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+	/*
+	 * Private data pointer for use by a restore module.  This can be used to
+	 * store state for the module that will be passed to each of its
+	 * callbacks.
+	 */
+	void	   *private_data;
+
+	/*
+	 * Indicates whether the callback that checks for timeline existence
+	 * merely copies the file.  If set to true, the server will verify that
+	 * the timeline history file exists after the callback returns.
+	 */
+	bool		timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init().  For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+									 const char *file, const char *path,
+									 const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+										  const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+										 const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+								  const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+							   const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+	RestoreStartupCB startup_cb;
+	RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+	RestoreWalSegmentCB restore_wal_segment_cb;
+	RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+	RestoreTimelineHistoryCB restore_timeline_history_cb;
+	TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+	TimelineHistoryExistsCB timeline_history_exists_cb;
+	ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+	ArchiveCleanupCB archive_cleanup_cb;
+	RecoveryEndConfiguredCB recovery_end_configured_cb;
+	RecoveryEndCB recovery_end_cb;
+	RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+	return RestoreCallbacks != NULL &&
+		RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+		RestoreCallbacks->restore_wal_segment_cb != NULL &&
+		RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+	return RestoreCallbacks != NULL &&
+		RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+		RestoreCallbacks->restore_timeline_history_cb != NULL &&
+		RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+	return RestoreCallbacks != NULL &&
+		RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+		RestoreCallbacks->timeline_history_exists_cb != NULL &&
+		RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+	return RestoreCallbacks != NULL &&
+		RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+		RestoreCallbacks->archive_cleanup_cb != NULL &&
+		RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+	return RestoreCallbacks != NULL &&
+		RestoreCallbacks->recovery_end_configured_cb != NULL &&
+		RestoreCallbacks->recovery_end_cb != NULL &&
+		RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif							/* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 570588e493..1eeae51d81 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
 #ifndef _SHELL_RESTORE_H
 #define _SHELL_RESTORE_H
 
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
-									  const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
 
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
 
 #endif							/* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 16220cfe44..c70e15aba7 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2362,6 +2362,8 @@ ResourceOwner
 ResourceReleaseCallback
 ResourceReleaseCallbackItem
 ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
 RestoreOptions
 RestorePass
 RestrictInfo
-- 
2.25.1

--SLDf9lqlvOQaIe6s--