public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v10 4/4] introduce restore_library
51+ messages / 7 participants
[nested] [flat]
* [PATCH v10 4/4] introduce restore_library
@ 2023-01-23 19:47 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-01-23 19:47 UTC (permalink / raw)
---
contrib/basic_wal_module/Makefile | 2 +
contrib/basic_wal_module/basic_wal_module.c | 69 ++++++-
contrib/basic_wal_module/meson.build | 5 +
contrib/basic_wal_module/t/001_restore.pl | 44 +++++
doc/src/sgml/backup.sgml | 43 ++++-
doc/src/sgml/basic-wal-module.sgml | 33 ++--
doc/src/sgml/config.sgml | 53 +++++-
doc/src/sgml/high-availability.sgml | 23 ++-
doc/src/sgml/wal-modules.sgml | 171 ++++++++++++++++--
src/backend/access/transam/shell_restore.c | 21 ++-
src/backend/access/transam/xlog.c | 13 +-
src/backend/access/transam/xlogarchive.c | 70 ++++++-
src/backend/access/transam/xlogrecovery.c | 26 ++-
src/backend/postmaster/checkpointer.c | 26 +++
src/backend/postmaster/startup.c | 23 ++-
src/backend/utils/misc/guc_tables.c | 10 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/access/xlog_internal.h | 1 +
src/include/access/xlogarchive.h | 44 ++++-
src/include/access/xlogrecovery.h | 1 +
20 files changed, 604 insertions(+), 75 deletions(-)
create mode 100644 contrib/basic_wal_module/t/001_restore.pl
diff --git a/contrib/basic_wal_module/Makefile b/contrib/basic_wal_module/Makefile
index 1f88aaf469..b92126ff52 100644
--- a/contrib/basic_wal_module/Makefile
+++ b/contrib/basic_wal_module/Makefile
@@ -9,6 +9,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_wal_module/basic_wal_mo
# which typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_wal_module/basic_wal_module.c b/contrib/basic_wal_module/basic_wal_module.c
index 78c36656a8..e17585cd93 100644
--- a/contrib/basic_wal_module/basic_wal_module.c
+++ b/contrib/basic_wal_module/basic_wal_module.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2023, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -30,6 +35,7 @@
#include <sys/time.h>
#include <unistd.h>
+#include "access/xlogarchive.h"
#include "common/int.h"
#include "miscadmin.h"
#include "postmaster/pgarch.h"
@@ -48,6 +54,8 @@ static bool basic_archive_file(const char *file, const char *path);
static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
+static bool basic_restore_file(const char *file, const char *path,
+ const char *lastRestartPointFileName);
/*
* _PG_init
@@ -58,7 +66,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_wal_module.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -87,6 +95,19 @@ _PG_archive_module_init(ArchiveModuleCallbacks *cb)
cb->archive_file_cb = basic_archive_file;
}
+/*
+ * _PG_recovery_module_init
+ *
+ * Returns the module's restore callback.
+ */
+void
+_PG_recovery_module_init(RecoveryModuleCallbacks *cb)
+{
+ AssertVariableIsOfType(&_PG_recovery_module_init, RecoveryModuleInit);
+
+ cb->restore_cb = basic_restore_file;
+}
+
/*
* check_archive_directory
*
@@ -99,8 +120,8 @@ check_archive_directory(char **newval, void **extra, GucSource source)
/*
* The default value is an empty string, so we have to accept that value.
- * Our check_configured callback also checks for this and prevents
- * archiving from proceeding if it is still empty.
+ * Our check_configured and restore callbacks also check for this and
+ * prevent archiving or recovery from proceeding if it is still empty.
*/
if (*newval == NULL || *newval[0] == '\0')
return true;
@@ -368,3 +389,45 @@ compare_files(const char *file1, const char *file2)
return ret;
}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file from the WAL archives.
+ */
+static bool
+basic_restore_file(const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ char source[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_wal_module", file)));
+
+ if (archive_directory == NULL || archive_directory[0] == '\0')
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"basic_wal_module.archive_directory\" is not set")));
+
+ /*
+ * Check whether the file exists. If not, we return false to indicate that
+ * there are no more files to restore.
+ */
+ snprintf(source, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(source, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m", source)));
+ return false;
+ }
+
+ copy_file(source, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_wal_module", file)));
+ return true;
+}
diff --git a/contrib/basic_wal_module/meson.build b/contrib/basic_wal_module/meson.build
index 59939d71c4..fe68a806a9 100644
--- a/contrib/basic_wal_module/meson.build
+++ b/contrib/basic_wal_module/meson.build
@@ -31,4 +31,9 @@ tests += {
# which typical runningcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_wal_module/t/001_restore.pl b/contrib/basic_wal_module/t/001_restore.pl
new file mode 100644
index 0000000000..c9f39ea413
--- /dev/null
+++ b/contrib/basic_wal_module/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_wal_module'");
+$node->append_conf('postgresql.conf', "basic_wal_module.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_wal_module'");
+$restore->append_conf('postgresql.conf', "basic_wal_module.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL was replayed
+my $result = $restore->safe_psql("postgres", "SELECT count(*) FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 12d9bb3f81..19f79fd2f2 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,27 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
- which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>
+ that defines a restore callback, which tells
+ <productname>PostgreSQL</productname> how to retrieve archived WAL file
+ segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ recovery modules can be more performant than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="wal-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1219,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore function returns
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore function provided by the
+ <varname>restore_library</varname> emits an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1232,7 +1256,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-wal-module.sgml b/doc/src/sgml/basic-wal-module.sgml
index f972566374..ebb9f0d8c3 100644
--- a/doc/src/sgml/basic-wal-module.sgml
+++ b/doc/src/sgml/basic-wal-module.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_wal_module</filename> is an example of an archive module.
- This module copies completed WAL segment files to the specified directory.
- This may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="wal-modules"/>.
+ <filename>basic_wal_module</filename> is an example of a write-ahead log
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and recovery modules. For
+ more information about write-ahead log modules, see
+ <xref linkend="wal-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a recovery module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-wal-module-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,7 +65,8 @@ basic_wal_module.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
- Server crashes may leave temporary files with the prefix
+ When <filename>basic_wal_module</filename> is used as an archive module,
+ server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
remove such files while the server is running as long as they are unrelated
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bba24bdcb8..feb4ca34af 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3807,7 +3807,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3835,7 +3836,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3870,7 +3872,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for recovery actions, including retrieving archived
+ segments of the WAL file series and executing tasks at restartpoints
+ and at recovery end. Either <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname> and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname> or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for recovery.
+ For more information, see <xref linkend="wal-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -3915,7 +3952,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -3944,7 +3984,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index f180607528..6266e2df7f 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,9 +639,11 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
- reaches the end of WAL available there and <varname>restore_command</varname>
- fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s restore callback.
+ Once it reaches the end of WAL available there and
+ <varname>restore_command</varname> or the restore callback fails, it tries
+ to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
from the last valid record found in archive or <filename>pg_wal</filename>. If that fails
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and restore callbacks provided by
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s
+ <function>archive_cleanup_cb</function> callback function to remove files
+ that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/doc/src/sgml/wal-modules.sgml b/doc/src/sgml/wal-modules.sgml
index 451c591e17..479b0a7d14 100644
--- a/doc/src/sgml/wal-modules.sgml
+++ b/doc/src/sgml/wal-modules.sgml
@@ -8,27 +8,33 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom WAL module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While
+ a shell command (e.g., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
When a custom <xref linkend="guc-archive-library"/> is configured, PostgreSQL
will submit completed WAL files to the module, and the server will avoid
recycling or removing these WAL files until the module indicates that the
- files were successfully archived. It is ultimately up to the module to
- decide what to do with each WAL file, but many recommendations are listed at
- <xref linkend="backup-archiving-wal"/>.
+ were successfully archived. When a custom
+ <xref linkend="guc-restore-library"/> is configured, PostgreSQL will use the
+ module for recovery actions. It is ultimately up to the module to decide how
+ to accomplish each task, but some recommendations are listed at
+ <xref linkend="backup-archiving-wal"/> and
+ <xref linkend="backup-pitr-recovery"/>.
</para>
<para>
- Archiving modules must at least consist of an initialization function (see
- <xref linkend="archive-module-init"/>) and the required callbacks (see
- <xref linkend="archive-module-callbacks"/>). However, archive modules are
- also permitted to do much more (e.g., declare GUCs and register background
- workers).
+ Write-ahead log modules must at least consist of an initialization function
+ (see <xref linkend="archive-module-init"/> and
+ <xref linkend="recovery-module-init"/>) and the required callbacks (see
+ <xref linkend="archive-module-callbacks"/> and
+ <xref linkend="recovery-module-callbacks"/>). However, write-ahead log
+ modules are also permitted to do much more (e.g., declare GUCs and register
+ background workers). A module may be used for both
+ <varname>archive_library</varname> and <varname>restore_library</varname>.
</para>
<para>
@@ -43,7 +49,7 @@
</indexterm>
<sect2 id="archive-module-init">
- <title>Initialization Functions</title>
+ <title>Archive Module Initialization Functions</title>
<indexterm zone="archive-module-init">
<primary>_PG_archive_module_init</primary>
</indexterm>
@@ -70,6 +76,12 @@ typedef void (*ArchiveModuleInit) (struct ArchiveModuleCallbacks *cb);
Only the <function>archive_file_cb</function> callback is required. The
others are optional.
</para>
+
+ <note>
+ <para>
+ <varname>archive_library</varname> is only loaded in the archiver process.
+ </para>
+ </note>
</sect2>
<sect2 id="archive-module-callbacks">
@@ -136,6 +148,139 @@ typedef bool (*ArchiveFileCB) (const char *file, const char *path);
<programlisting>
typedef void (*ArchiveShutdownCB) (void);
+</programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
+
+ <sect1 id="recovery-modules">
+ <title>Recovery Modules</title>
+ <indexterm zone="recovery-modules">
+ <primary>Recovery Modules</primary>
+ </indexterm>
+
+ <sect2 id="recovery-module-init">
+ <title>Recovery Module Initialization Functions</title>
+ <indexterm zone="recovery-module-init">
+ <primary>_PG_recovery_module_init</primary>
+ </indexterm>
+ <para>
+ A recovery library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/> as the library base name. The
+ normal library search path is used to locate the library. To provide the
+ required recovery module callbacks and to indicate that the library is
+ actually a recovery module, it needs to provide a function named
+ <function>_PG_recovery_module_init</function>. This function is passed a
+ struct that needs to be filled with the callback function pointers for
+ individual actions.
+
+<programlisting>
+typedef struct RecoveryModuleCallbacks
+{
+ RecoveryRestoreCB restore_cb;
+ RecoveryArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndCB recovery_end_cb;
+ RecoveryShutdownCB shutdown_cb;
+} RecoveryModuleCallbacks;
+typedef void (*RecoveryModuleInit) (struct RecoveryModuleCallbacks *cb);
+</programlisting>
+
+ The <function>restore_cb</function> callback is required for archive
+ recovery, but it is optional for streaming replication. The others are
+ always optional.
+ </para>
+
+ <note>
+ <para>
+ <varname>restore_library</varname> is only loaded in the startup and
+ checkpointer processes and in single-user mode.
+ </para>
+ </note>
+ </sect2>
+
+ <sect2 id="recovery-module-callbacks">
+ <title>Recovery Module Callbacks</title>
+ <para>
+ The recovery callbacks define the actual behavior of the module. The
+ server will call them as required to execute recovery actions.
+ </para>
+
+ <sect3 id="recovery-module-restore">
+ <title>Restore Callback</title>
+ <para>
+ The <function>restore_cb</function> callback is called to retrieve a
+ single archived segment of the WAL file series for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RecoveryRestoreCB) (const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="recovery-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point and is intended to provide a mechanism for cleaning up old
+ archived WAL files that are no longer needed by the standby server.
+
+<programlisting>
+typedef void (*RecoveryArchiveCleanupCB) (const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="recovery-module-restore"><function>restore_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="recovery-module-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="recovery-module-restore"><function>restore_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="recovery-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the recovery module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations.
+
+<programlisting>
+typedef void (*RecoveryShutdownCB) (void);
</programlisting>
</para>
</sect3>
diff --git a/src/backend/access/transam/shell_restore.c b/src/backend/access/transam/shell_restore.c
index 8458209f49..dfd409905c 100644
--- a/src/backend/access/transam/shell_restore.c
+++ b/src/backend/access/transam/shell_restore.c
@@ -4,7 +4,8 @@
* Recovery functions for a user-specified shell command.
*
* These recovery functions use a user-specified shell command (e.g. based
- * on the GUC restore_command).
+ * on the GUC restore_command). It is used as the default, but other
+ * modules may define their own recovery logic.
*
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
@@ -24,6 +25,10 @@
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static bool shell_restore(const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static void shell_archive_cleanup(const char *lastRestartPointFileName);
+static void shell_recovery_end(const char *lastRestartPointFileName);
static bool ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
@@ -31,6 +36,16 @@ static bool ExecuteRecoveryCommand(const char *command,
uint32 wait_event_info,
int fail_elevel);
+void
+shell_restore_init(RecoveryModuleCallbacks *cb)
+{
+ AssertVariableIsOfType(&shell_restore_init, RecoveryModuleInit);
+
+ cb->restore_cb = shell_restore;
+ cb->archive_cleanup_cb = shell_archive_cleanup;
+ cb->recovery_end_cb = shell_recovery_end;
+}
+
/*
* Attempt to execute a shell-based restore command.
*
@@ -88,7 +103,7 @@ shell_restore(const char *file, const char *path,
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
+static void
shell_archive_cleanup(const char *lastRestartPointFileName)
{
char *cmd;
@@ -104,7 +119,7 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
+static void
shell_recovery_end(const char *lastRestartPointFileName)
{
char *cmd;
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index fb4c860bde..0e6d2d9363 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -4886,15 +4886,16 @@ static void
CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
+
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*/
- if (recoveryEndCommand && strcmp(recoveryEndCommand, "") != 0)
+ if (RecoveryContext.recovery_end_cb)
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RecoveryContext.recovery_end_cb(lastRestartPointFname);
}
/*
@@ -7309,14 +7310,14 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Execute the archive-cleanup callback, if any.
*/
- if (archiveCleanupCommand && strcmp(archiveCleanupCommand, "") != 0)
+ if (RecoveryContext.archive_cleanup_cb)
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RecoveryContext.archive_cleanup_cb(lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 4b89addf97..4af5689c25 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -22,6 +22,8 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
+#include "access/xlogrecovery.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
@@ -31,6 +33,11 @@
#include "storage/ipc.h"
#include "storage/lwlock.h"
+/*
+ * Global context for recovery-related callbacks.
+ */
+RecoveryModuleCallbacks RecoveryContext;
+
/*
* Attempt to retrieve the specified file from off-line archival storage.
* If successful, fill "path" with its complete path (note that this will be
@@ -70,7 +77,7 @@ RestoreArchivedFile(char *path, const char *xlogfname,
goto not_available;
/* In standby mode, restore_command might not be supplied */
- if (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)
+ if (RecoveryContext.restore_cb == NULL)
goto not_available;
/*
@@ -148,14 +155,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);
/*
- * Check signals before restore command and reset afterwards.
+ * Check signals before restore callback and reset afterwards.
*/
PreRestoreCommand();
/*
* Copy xlog from archival storage to XLOGDIR
*/
- ret = shell_restore(xlogfname, xlogpath, lastRestartPointFname);
+ ret = RecoveryContext.restore_cb(xlogfname, xlogpath,
+ lastRestartPointFname);
PostRestoreCommand();
@@ -602,3 +610,59 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the recovery callbacks into our global RecoveryContext. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRecoveryCallbacks(void)
+{
+ RecoveryModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_recovery_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RecoveryModuleInit)
+ load_external_function(restoreLibrary, "_PG_recovery_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("recovery modules have to define the symbol "
+ "_PG_recovery_module_init")));
+
+ memset(&RecoveryContext, 0, sizeof(RecoveryModuleCallbacks));
+ (*init) (&RecoveryContext);
+
+ /*
+ * If using shell commands, remove callbacks for any commands that are not
+ * set.
+ */
+ if (restoreLibrary[0] == '\0')
+ {
+ if (recoveryRestoreCommand[0] == '\0')
+ RecoveryContext.restore_cb = NULL;
+ if (archiveCleanupCommand[0] == '\0')
+ RecoveryContext.archive_cleanup_cb = NULL;
+ if (recoveryEndCommand[0] == '\0')
+ RecoveryContext.recovery_end_cb = NULL;
+ }
+}
+
+/*
+ * Call the shutdown callback of the loaded recovery module, if defined.
+ */
+void
+call_recovery_module_shutdown_cb(int code, Datum arg)
+{
+ if (RecoveryContext.shutdown_cb)
+ RecoveryContext.shutdown_cb();
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 2a5352f879..04f94e4d53 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -80,6 +80,7 @@ const struct config_enum_entry recovery_target_action_options[] = {
/* options formerly taken from recovery.conf for archive recovery */
char *recoveryRestoreCommand = NULL;
+char *restoreLibrary = NULL;
char *recoveryEndCommand = NULL;
char *archiveCleanupCommand = NULL;
RecoveryTargetType recoveryTarget = RECOVERY_TARGET_UNSET;
@@ -1053,24 +1054,37 @@ validateRecoveryParameters(void)
if (!ArchiveRecoveryRequested)
return;
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command");
+ CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command");
+ before_shmem_exit(call_recovery_module_shutdown_cb, 0);
+ LoadRecoveryCallbacks();
+
/*
* Check for compulsory parameters
*/
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ RecoveryContext.restore_cb == NULL)
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
- errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
+ (errmsg("specified neither primary_conninfo nor restore_command "
+ "nor a restore_library that defines a restore callback"),
+ errhint("The database server will regularly poll the pg_wal "
+ "subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (RecoveryContext.restore_cb == NULL)
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a restore_library that defines "
+ "a restore callback when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index de0bbbfa79..6350fd0b83 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -38,6 +38,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
+#include "access/xlogarchive.h"
#include "access/xlogrecovery.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
@@ -222,6 +223,16 @@ CheckpointerMain(void)
*/
before_shmem_exit(pgstat_before_server_shutdown, 0);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks. We do this before setting up the exception handler
+ * so that any problems result in a server crash shortly after startup.
+ */
+ CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command");
+ before_shmem_exit(call_recovery_module_shutdown_cb, 0);
+ LoadRecoveryCallbacks();
+
/*
* Create a memory context that we will do all our work in. We do this so
* that we can reset the context during error recovery and thereby avoid
@@ -548,6 +559,9 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -563,6 +577,18 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command");
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ call_recovery_module_shutdown_cb(0, (Datum) 0);
+ LoadRecoveryCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index 8786186898..f9ff2b5583 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -20,6 +20,7 @@
#include "postgres.h"
#include "access/xlog.h"
+#include "access/xlogarchive.h"
#include "access/xlogrecovery.h"
#include "access/xlogutils.h"
#include "libpq/pqsignal.h"
@@ -133,13 +134,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the recovery callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -161,6 +166,22 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command");
+ CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command");
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_recovery_module_shutdown_cb(0, (Datum) 0);
+ LoadRecoveryCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 4ac808ed22..06d36513aa 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3787,6 +3787,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for recovery actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d06074b86f..5641e5a3a8 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -270,6 +270,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for recovery actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 59fc7bc105..756f0898b5 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -400,5 +400,6 @@ extern PGDLLIMPORT bool ArchiveRecoveryRequested;
extern PGDLLIMPORT bool InArchiveRecovery;
extern PGDLLIMPORT bool StandbyMode;
extern PGDLLIMPORT char *recoveryRestoreCommand;
+extern PGDLLIMPORT char *restoreLibrary;
#endif /* XLOG_INTERNAL_H */
diff --git a/src/include/access/xlogarchive.h b/src/include/access/xlogarchive.h
index 299304703e..71c9b88165 100644
--- a/src/include/access/xlogarchive.h
+++ b/src/include/access/xlogarchive.h
@@ -30,9 +30,45 @@ extern bool XLogArchiveIsReady(const char *xlog);
extern bool XLogArchiveIsReadyOrDone(const char *xlog);
extern void XLogArchiveCleanup(const char *xlog);
-extern bool shell_restore(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Recovery module callbacks
+ *
+ * These callback functions should be defined by recovery libraries and
+ * returned via _PG_recovery_module_init(). For more information about the
+ * purpose of each callback, refer to the recovery modules documentation.
+ */
+typedef bool (*RecoveryRestoreCB) (const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef void (*RecoveryArchiveCleanupCB) (const char *lastRestartPointFileName);
+typedef void (*RecoveryEndCB) (const char *lastRestartPointFileName);
+typedef void (*RecoveryShutdownCB) (void);
+
+typedef struct RecoveryModuleCallbacks
+{
+ RecoveryRestoreCB restore_cb;
+ RecoveryArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndCB recovery_end_cb;
+ RecoveryShutdownCB shutdown_cb;
+} RecoveryModuleCallbacks;
+
+extern RecoveryModuleCallbacks RecoveryContext;
+
+/*
+ * Type of the shared library symbol _PG_recovery_module_init that is looked up
+ * when loading a recovery library.
+ */
+typedef void (*RecoveryModuleInit) (RecoveryModuleCallbacks *cb);
+
+extern PGDLLEXPORT void _PG_recovery_module_init(RecoveryModuleCallbacks *cb);
+
+extern void LoadRecoveryCallbacks(void);
+extern void call_recovery_module_shutdown_cb(int code, Datum arg);
+
+/*
+ * Since the logic for recovery via a shell command is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern void shell_restore_init(RecoveryModuleCallbacks *cb);
#endif /* XLOG_ARCHIVE_H */
diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h
index 47c29350f5..35d1d09374 100644
--- a/src/include/access/xlogrecovery.h
+++ b/src/include/access/xlogrecovery.h
@@ -55,6 +55,7 @@ extern PGDLLIMPORT int recovery_min_apply_delay;
extern PGDLLIMPORT char *PrimaryConnInfo;
extern PGDLLIMPORT char *PrimarySlotName;
extern PGDLLIMPORT char *recoveryRestoreCommand;
+extern PGDLLIMPORT char *restoreLibrary;
extern PGDLLIMPORT char *recoveryEndCommand;
extern PGDLLIMPORT char *archiveCleanupCommand;
--
2.25.1
--7JfCtLOvnd9MIVvH--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v18 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b38cbd714a..bba79e19ab 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..f811398fa2 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 320fa857ea..8ebbd6ce50 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bfa569502..4afdcc3aad 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1114,18 +1115,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 46197d56f8..47993aa35e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -224,6 +225,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -556,6 +576,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -571,6 +595,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index e391bf6aa6..3cbd382e00 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -25,6 +25,7 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -118,13 +119,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -146,6 +151,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -255,6 +284,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 45013582a7..724d268d7b 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index edcc0282b2..2939042b06 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 3a02310f26..6a05313f6b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2386,6 +2386,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--IJpNTDwzlM2Ie8A6--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v14 7/7] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 1 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 28 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 53 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 142 +++++++
src/include/restore/shell_restore.h | 16 +-
19 files changed, 1103 insertions(+), 88 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..d12d45e0d2 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -5,6 +5,7 @@ PGFILEDESC = "basic_archive - basic archive module"
REGRESS = basic_archive
REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+TAP_TESTS = 1
# Disabled because these tests require "shared_preload_libraries=basic_archive",
# which typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index cd852888ce..b04d83da4c 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2023, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -426,3 +468,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index bc1380e6f6..4e9f5002de 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# which typical runningcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..f7fa124695
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index f30ee591e7..56373a211b 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -167,4 +169,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index be05a33205..5555f23a85 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1218,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1232,7 +1255,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e5c41cc6c6..288d58f48a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3807,7 +3807,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3835,7 +3836,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3870,7 +3872,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -3915,7 +3952,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -3944,7 +3984,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 9d0deaeeb8..094140cecc 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8e1e71c256..f06385bea5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,7 +84,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -4891,18 +4891,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7317,18 +7318,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 4b45ea8753..50bcda1120 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,15 +23,22 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -75,15 +82,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -175,14 +182,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -190,6 +200,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -631,3 +651,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f0e1007f92..c6645c2413 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -50,6 +50,7 @@
#include "postmaster/startup.h"
#include "replication/slot.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1078,18 +1079,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index aaad5c5228..73ff592f9f 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -233,6 +234,24 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback, if
+ * one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking checkpoints
+ * or shutting down the server when the parameters are misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -548,6 +567,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -563,6 +586,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides restarting
+ * the process, and there should be little harm in leaving it around,
+ * so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index adce0ffcef..a096b79182 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -29,6 +29,7 @@
#include "pgstat.h"
#include "postmaster/interrupt.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/pmsignal.h"
@@ -148,13 +149,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -176,6 +181,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded shutdown
+ * callback and load the new callbacks. There's presently no good way to
+ * unload a library besides restarting the process, and there should be
+ * little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -285,6 +314,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index fcd5475c88..b67cd4ac48 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2023, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -153,8 +206,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -162,8 +215,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -173,8 +227,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -182,8 +236,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1c0583fe26..a42819399b 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -63,6 +63,7 @@
#include "replication/logicallauncher.h"
#include "replication/slot.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3788,6 +3789,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d06074b86f..53757fea5f 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -270,6 +270,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..c0aef33ba5
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence merely
+ * copies the file. If set to true, the server will verify that the
+ * timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 570588e493..1eeae51d81 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
--
2.25.1
--IS0zKkzwUGydFO0o--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v20 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..9a2570f87f 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a86a9fe93a 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 5756f52124..3a05581495 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/pgarch.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 07213b1897..1bf0d0ea11 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1117,18 +1118,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 8ef600ae72..c31d0f3bd2 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -46,6 +46,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -230,6 +231,25 @@ CheckpointerMain(char *startup_data, size_t startup_data_len)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -562,6 +582,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -577,6 +601,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index cc665ce259..f8d3d6b3d2 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -26,6 +26,7 @@
#include "miscadmin.h"
#include "postmaster/auxprocess.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -119,13 +120,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -147,6 +152,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -261,6 +290,19 @@ StartupProcessMain(char *startup_data, size_t startup_data_len)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1e71e7db4a..f84724a0ed 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..5265857519 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index a40c59b2e0..db4cc68a88 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2405,6 +2405,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--3V7upXqbjpZ4EhLz--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v16 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 1 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1103 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..d12d45e0d2 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -5,6 +5,7 @@ PGFILEDESC = "basic_archive - basic archive module"
REGRESS = basic_archive
REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+TAP_TESTS = 1
# Disabled because these tests require "shared_preload_libraries=basic_archive",
# which typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 4d78c31859..862689f981 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2023, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -426,3 +468,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index bc1380e6f6..4e9f5002de 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# which typical runningcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..f7fa124695
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index 7ed50a3b52..bf2fdb313a 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -166,4 +168,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 8cb24d6ae5..4b20651234 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1218,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1232,7 +1255,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 3839c72c86..cdc2f902e0 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3751,7 +3751,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3779,7 +3780,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3814,7 +3816,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -3859,7 +3896,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -3888,7 +3928,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 5f9257313a..09705a14a2 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1307748b4e..c79547e403 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,7 +84,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -4983,18 +4983,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7453,18 +7454,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index a925ac22f6..316b866af0 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,15 +23,22 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -75,15 +82,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -175,14 +182,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -190,6 +200,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -631,3 +651,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index d43c229eca..8d231dcc82 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -50,6 +50,7 @@
#include "postmaster/startup.h"
#include "replication/slot.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1077,18 +1078,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index ace9893d95..4ded7f3ff3 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -233,6 +234,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -547,6 +567,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -562,6 +586,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index adce0ffcef..8ad570c725 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -29,6 +29,7 @@
#include "pgstat.h"
#include "postmaster/interrupt.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/pmsignal.h"
@@ -148,13 +149,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -176,6 +181,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -285,6 +314,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index bc1f4a431a..ce77fde66b 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2023, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 4c58574166..21e91908aa 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -67,6 +67,7 @@
#include "replication/logicallauncher.h"
#include "replication/slot.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3828,6 +3829,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d08d55c3fe..2f9ad7e1bd 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -272,6 +272,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..b9a5810aa8
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 570588e493..1eeae51d81 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 16220cfe44..c70e15aba7 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2362,6 +2362,8 @@ ResourceOwner
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--SLDf9lqlvOQaIe6s--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v20 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..9a2570f87f 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a86a9fe93a 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 5756f52124..3a05581495 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/pgarch.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 07213b1897..1bf0d0ea11 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1117,18 +1118,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 8ef600ae72..c31d0f3bd2 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -46,6 +46,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -230,6 +231,25 @@ CheckpointerMain(char *startup_data, size_t startup_data_len)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -562,6 +582,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -577,6 +601,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index cc665ce259..f8d3d6b3d2 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -26,6 +26,7 @@
#include "miscadmin.h"
#include "postmaster/auxprocess.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -119,13 +120,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -147,6 +152,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -261,6 +290,19 @@ StartupProcessMain(char *startup_data, size_t startup_data_len)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1e71e7db4a..f84724a0ed 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..5265857519 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index a40c59b2e0..db4cc68a88 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2405,6 +2405,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--3V7upXqbjpZ4EhLz--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v13 7/7] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 1 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 28 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 53 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 142 +++++++
src/include/restore/shell_restore.h | 16 +-
19 files changed, 1103 insertions(+), 88 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..d12d45e0d2 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -5,6 +5,7 @@ PGFILEDESC = "basic_archive - basic archive module"
REGRESS = basic_archive
REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+TAP_TESTS = 1
# Disabled because these tests require "shared_preload_libraries=basic_archive",
# which typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index cd852888ce..b04d83da4c 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2023, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -426,3 +468,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index bc1380e6f6..4e9f5002de 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# which typical runningcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..85b14c5113
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL was replayed
+my $result = $restore->safe_psql("postgres", "SELECT count(*) FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index f30ee591e7..56373a211b 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -167,4 +169,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index be05a33205..5555f23a85 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1218,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1232,7 +1255,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e5c41cc6c6..288d58f48a 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3807,7 +3807,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3835,7 +3836,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3870,7 +3872,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -3915,7 +3952,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -3944,7 +3984,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 9d0deaeeb8..094140cecc 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8e1e71c256..f06385bea5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,7 +84,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -4891,18 +4891,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7317,18 +7318,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 4b45ea8753..50bcda1120 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,15 +23,22 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -75,15 +82,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -175,14 +182,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -190,6 +200,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -631,3 +651,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f0e1007f92..c6645c2413 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -50,6 +50,7 @@
#include "postmaster/startup.h"
#include "replication/slot.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1078,18 +1079,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index aaad5c5228..73ff592f9f 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -233,6 +234,24 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback, if
+ * one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking checkpoints
+ * or shutting down the server when the parameters are misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -548,6 +567,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -563,6 +586,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides restarting
+ * the process, and there should be little harm in leaving it around,
+ * so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index adce0ffcef..a096b79182 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -29,6 +29,7 @@
#include "pgstat.h"
#include "postmaster/interrupt.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/pmsignal.h"
@@ -148,13 +149,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -176,6 +181,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded shutdown
+ * callback and load the new callbacks. There's presently no good way to
+ * unload a library besides restarting the process, and there should be
+ * little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -285,6 +314,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index fcd5475c88..b67cd4ac48 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2023, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -153,8 +206,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -162,8 +215,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -173,8 +227,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -182,8 +236,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1c0583fe26..a42819399b 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -63,6 +63,7 @@
#include "replication/logicallauncher.h"
#include "replication/slot.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3788,6 +3789,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d06074b86f..53757fea5f 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -270,6 +270,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..c0aef33ba5
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence merely
+ * copies the file. If set to true, the server will verify that the
+ * timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 570588e493..1eeae51d81 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
--
2.25.1
--5vNYLRcllDrimb99--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v18 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b38cbd714a..bba79e19ab 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..f811398fa2 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 320fa857ea..8ebbd6ce50 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bfa569502..4afdcc3aad 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1114,18 +1115,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 46197d56f8..47993aa35e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -224,6 +225,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -556,6 +576,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -571,6 +595,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index e391bf6aa6..3cbd382e00 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -25,6 +25,7 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -118,13 +119,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -146,6 +151,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -255,6 +284,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 45013582a7..724d268d7b 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index edcc0282b2..2939042b06 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 3a02310f26..6a05313f6b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2386,6 +2386,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--IJpNTDwzlM2Ie8A6--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v19 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..9a2570f87f 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a86a9fe93a 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 5756f52124..3a05581495 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/pgarch.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 07213b1897..1bf0d0ea11 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1117,18 +1118,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 46197d56f8..47993aa35e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -224,6 +225,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -556,6 +576,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -571,6 +595,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index e391bf6aa6..3cbd382e00 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -25,6 +25,7 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -118,13 +119,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -146,6 +151,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -255,6 +284,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 57d9de4dd9..295cde9247 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..5265857519 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 2f975ac53a..8e190a0c79 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2388,6 +2388,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--oyUTqETQ0mS9luUI--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v12 6/6] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 1 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 28 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 53 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 142 +++++++
src/include/restore/shell_restore.h | 16 +-
19 files changed, 1103 insertions(+), 88 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..d12d45e0d2 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -5,6 +5,7 @@ PGFILEDESC = "basic_archive - basic archive module"
REGRESS = basic_archive
REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+TAP_TESTS = 1
# Disabled because these tests require "shared_preload_libraries=basic_archive",
# which typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index cd852888ce..b04d83da4c 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2023, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -426,3 +468,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index bc1380e6f6..4e9f5002de 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# which typical runningcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..85b14c5113
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL was replayed
+my $result = $restore->safe_psql("postgres", "SELECT count(*) FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index f30ee591e7..56373a211b 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -167,4 +169,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index be05a33205..5555f23a85 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1218,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1232,7 +1255,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ecd9aa73ef..8fec106a24 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3807,7 +3807,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3835,7 +3836,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3870,7 +3872,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -3915,7 +3952,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -3944,7 +3984,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index f180607528..957d427616 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c4268af688..b6fa39c7cf 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,7 +84,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -4888,18 +4888,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7314,18 +7315,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 4b45ea8753..50bcda1120 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,15 +23,22 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -75,15 +82,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -175,14 +182,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -190,6 +200,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -631,3 +651,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index f0e1007f92..c6645c2413 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -50,6 +50,7 @@
#include "postmaster/startup.h"
#include "replication/slot.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1078,18 +1079,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index aaad5c5228..73ff592f9f 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -233,6 +234,24 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback, if
+ * one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking checkpoints
+ * or shutting down the server when the parameters are misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -548,6 +567,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -563,6 +586,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides restarting
+ * the process, and there should be little harm in leaving it around,
+ * so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index 4648299ced..91229d96be 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -29,6 +29,7 @@
#include "pgstat.h"
#include "postmaster/interrupt.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/pmsignal.h"
@@ -151,13 +152,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -179,6 +184,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded shutdown
+ * callback and load the new callbacks. There's presently no good way to
+ * unload a library besides restarting the process, and there should be
+ * little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -288,6 +317,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index fcd5475c88..b67cd4ac48 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2023, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -153,8 +206,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -162,8 +215,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -173,8 +227,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -182,8 +236,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 1c0583fe26..a42819399b 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -63,6 +63,7 @@
#include "replication/logicallauncher.h"
#include "replication/slot.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3788,6 +3789,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index d06074b86f..53757fea5f 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -270,6 +270,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..c0aef33ba5
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence merely
+ * copies the file. If set to true, the server will verify that the
+ * timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 570588e493..1eeae51d81 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
--
2.25.1
--YiEDa0DAkWCtVeE4--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v19 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..9a2570f87f 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a86a9fe93a 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 5756f52124..3a05581495 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/pgarch.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 07213b1897..1bf0d0ea11 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1117,18 +1118,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 46197d56f8..47993aa35e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -224,6 +225,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -556,6 +576,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -571,6 +595,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index e391bf6aa6..3cbd382e00 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -25,6 +25,7 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -118,13 +119,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -146,6 +151,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -255,6 +284,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 57d9de4dd9..295cde9247 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..5265857519 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 2f975ac53a..8e190a0c79 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2388,6 +2388,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--oyUTqETQ0mS9luUI--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v19 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 65a6e6c408..9a2570f87f 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index b48209fc2f..a86a9fe93a 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 5756f52124..3a05581495 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/pgarch.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 07213b1897..1bf0d0ea11 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1117,18 +1118,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 46197d56f8..47993aa35e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -224,6 +225,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -556,6 +576,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -571,6 +595,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index e391bf6aa6..3cbd382e00 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -25,6 +25,7 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -118,13 +119,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -146,6 +151,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -255,6 +284,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 57d9de4dd9..295cde9247 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2244ee52f7..5265857519 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 2f975ac53a..8e190a0c79 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2388,6 +2388,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--oyUTqETQ0mS9luUI--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v15 7/7] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 1 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 28 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 53 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 142 +++++++
src/include/restore/shell_restore.h | 16 +-
19 files changed, 1103 insertions(+), 88 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 55d299d650..d12d45e0d2 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -5,6 +5,7 @@ PGFILEDESC = "basic_archive - basic archive module"
REGRESS = basic_archive
REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf
+TAP_TESTS = 1
# Disabled because these tests require "shared_preload_libraries=basic_archive",
# which typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index cd852888ce..b04d83da4c 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2023, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -426,3 +468,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index bc1380e6f6..4e9f5002de 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# which typical runningcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..f7fa124695
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2023, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index 7ed50a3b52..bf2fdb313a 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -166,4 +168,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 8cb24d6ae5..4b20651234 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1180,9 +1180,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1201,14 +1218,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1232,7 +1255,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 091a79d4f3..4b0dd21108 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3873,7 +3873,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3901,7 +3902,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3936,7 +3938,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -3981,7 +4018,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4010,7 +4050,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 9d0deaeeb8..094140cecc 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eaba35b59e..79d33e565a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -84,7 +84,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -4904,18 +4904,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7334,18 +7335,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during log
- * shipping replication. All files earlier than this point can be deleted
- * from the archive, though there is no requirement to do so.
+ * The callback is provided the archive file cutoff point for use during
+ * log shipping replication. All files earlier than this point can be
+ * deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index a925ac22f6..b1354dcee9 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,15 +23,22 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lwlock.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -75,15 +82,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -175,14 +182,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -190,6 +200,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -631,3 +651,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index e3b02ed760..9153e7bac0 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -50,6 +50,7 @@
#include "postmaster/startup.h"
#include "replication/slot.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1078,18 +1079,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index ace9893d95..403f4051c0 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -233,6 +234,24 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback, if
+ * one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking checkpoints
+ * or shutting down the server when the parameters are misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -547,6 +566,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -562,6 +585,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides restarting
+ * the process, and there should be little harm in leaving it around,
+ * so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index adce0ffcef..a096b79182 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -29,6 +29,7 @@
#include "pgstat.h"
#include "postmaster/interrupt.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/pmsignal.h"
@@ -148,13 +149,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -176,6 +181,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded shutdown
+ * callback and load the new callbacks. There's presently no good way to
+ * unload a library besides restarting the process, and there should be
+ * little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -285,6 +314,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index fcd5475c88..b67cd4ac48 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2023, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -153,8 +206,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -162,8 +215,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -173,8 +227,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -182,8 +236,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index cab3ddbe11..81e4d6d183 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -65,6 +65,7 @@
#include "replication/logicallauncher.h"
#include "replication/slot.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3839,6 +3840,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index dce5049bc2..5b96fe5ea8 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -275,6 +275,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..c0aef33ba5
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2023, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence merely
+ * copies the file. If set to true, the server will verify that the
+ * timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 570588e493..1eeae51d81 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
--
2.25.1
--k+w/mQv8wyuph6w0--
^ permalink raw reply [nested|flat] 51+ messages in thread
* [PATCH v18 5/5] introduce restore_library
@ 2023-02-16 06:06 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Nathan Bossart @ 2023-02-16 06:06 UTC (permalink / raw)
---
contrib/basic_archive/Makefile | 2 +
contrib/basic_archive/basic_archive.c | 153 +++++++-
contrib/basic_archive/meson.build | 5 +
contrib/basic_archive/t/001_restore.pl | 44 +++
doc/src/sgml/archive-and-restore-modules.sgml | 356 +++++++++++++++++-
doc/src/sgml/backup.sgml | 40 +-
doc/src/sgml/basic-archive.sgml | 31 +-
doc/src/sgml/config.sgml | 53 ++-
doc/src/sgml/high-availability.sgml | 18 +-
src/backend/access/transam/xlog.c | 20 +-
src/backend/access/transam/xlogarchive.c | 96 ++++-
src/backend/access/transam/xlogrecovery.c | 14 +-
src/backend/postmaster/checkpointer.c | 54 +++
src/backend/postmaster/startup.c | 44 ++-
src/backend/restore/shell_restore.c | 85 ++++-
src/backend/utils/misc/guc_tables.c | 11 +
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/restore/restore_module.h | 143 +++++++
src/include/restore/shell_restore.h | 16 +-
src/tools/pgindent/typedefs.list | 2 +
20 files changed, 1104 insertions(+), 84 deletions(-)
create mode 100644 contrib/basic_archive/t/001_restore.pl
create mode 100644 src/include/restore/restore_module.h
diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile
index 100ed81f12..e61f7d676f 100644
--- a/contrib/basic_archive/Makefile
+++ b/contrib/basic_archive/Makefile
@@ -10,6 +10,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c
# typical installcheck users do not have (e.g. buildfarm clients).
NO_INSTALLCHECK = 1
+TAP_TESTS = 1
+
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c
index 6b102e9072..e0d8dce364 100644
--- a/contrib/basic_archive/basic_archive.c
+++ b/contrib/basic_archive/basic_archive.c
@@ -17,6 +17,11 @@
* a file is successfully archived and then the system crashes before
* a durable record of the success has been made.
*
+ * This file also demonstrates a basic restore library implementation that
+ * is roughly equivalent to the following shell command:
+ *
+ * cp /path/to/archivedir/%f %p
+ *
* Copyright (c) 2022-2024, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -33,6 +38,7 @@
#include "archive/archive_module.h"
#include "common/int.h"
#include "miscadmin.h"
+#include "restore/restore_module.h"
#include "storage/copydir.h"
#include "storage/fd.h"
#include "utils/guc.h"
@@ -54,6 +60,16 @@ static void basic_archive_file_internal(const char *file, const char *path);
static bool check_archive_directory(char **newval, void **extra, GucSource source);
static bool compare_files(const char *file1, const char *file2);
static void basic_archive_shutdown(ArchiveModuleState *state);
+static bool basic_restore_configured(RestoreModuleState *state);
+static bool basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool basic_restore_file(const char *file, const char *path);
+static bool basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file,
+ const char *path);
static const ArchiveModuleCallbacks basic_archive_callbacks = {
.startup_cb = basic_archive_startup,
@@ -62,6 +78,21 @@ static const ArchiveModuleCallbacks basic_archive_callbacks = {
.shutdown_cb = basic_archive_shutdown
};
+static const RestoreModuleCallbacks basic_restore_callbacks = {
+ .startup_cb = NULL,
+ .restore_wal_segment_configured_cb = basic_restore_configured,
+ .restore_wal_segment_cb = basic_restore_wal_segment,
+ .restore_timeline_history_configured_cb = basic_restore_configured,
+ .restore_timeline_history_cb = basic_restore_timeline_history,
+ .timeline_history_exists_configured_cb = basic_restore_configured,
+ .timeline_history_exists_cb = basic_restore_timeline_history_exists,
+ .archive_cleanup_configured_cb = NULL,
+ .archive_cleanup_cb = NULL,
+ .recovery_end_configured_cb = NULL,
+ .recovery_end_cb = NULL,
+ .shutdown_cb = NULL
+};
+
/*
* _PG_init
*
@@ -71,7 +102,7 @@ void
_PG_init(void)
{
DefineCustomStringVariable("basic_archive.archive_directory",
- gettext_noop("Archive file destination directory."),
+ gettext_noop("Archive file source/destination directory."),
NULL,
&archive_directory,
"",
@@ -93,6 +124,17 @@ _PG_archive_module_init(void)
return &basic_archive_callbacks;
}
+/*
+ * _PG_restore_module_init
+ *
+ * Returns the module's restore callbacks.
+ */
+const RestoreModuleCallbacks *
+_PG_restore_module_init(void)
+{
+ return &basic_restore_callbacks;
+}
+
/*
* basic_archive_startup
*
@@ -431,3 +473,112 @@ basic_archive_shutdown(ArchiveModuleState *state)
pfree(data);
state->private_data = NULL;
}
+
+/*
+ * basic_restore_configured
+ *
+ * Checks that archive_directory is not blank.
+ */
+static bool
+basic_restore_configured(RestoreModuleState *state)
+{
+ return archive_directory != NULL && archive_directory[0] != '\0';
+}
+
+/*
+ * basic_restore_wal_segment
+ *
+ * Retrieves one archived WAL segment.
+ */
+static bool
+basic_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_timeline_history
+ *
+ * Retrieves one timeline history file.
+ */
+static bool
+basic_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ return basic_restore_file(file, path);
+}
+
+/*
+ * basic_restore_file
+ *
+ * Retrieves one file.
+ */
+static bool
+basic_restore_file(const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("restoring \"%s\" via basic_archive",
+ file)));
+
+ /*
+ * If the file doesn't exist, return false to indicate that there are no
+ * more files to restore.
+ */
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ copy_file(archive_path, path);
+
+ ereport(DEBUG1,
+ (errmsg("restored \"%s\" via basic_archive",
+ file)));
+ return true;
+}
+
+/*
+ * basic_restore_timeline_history_exists
+ *
+ * Check whether a timeline history file is present in the archive directory.
+ */
+static bool
+basic_restore_timeline_history_exists(RestoreModuleState *state,
+ const char *file, const char *path)
+{
+ char archive_path[MAXPGPATH];
+ struct stat st;
+
+ ereport(DEBUG1,
+ (errmsg("checking existence of \"%s\" via basic_archive",
+ file)));
+
+ snprintf(archive_path, MAXPGPATH, "%s/%s", archive_directory, file);
+ if (stat(archive_path, &st) != 0)
+ {
+ int elevel = (errno == ENOENT) ? DEBUG1 : ERROR;
+
+ ereport(elevel,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m",
+ archive_path)));
+ return false;
+ }
+
+ ereport(DEBUG1,
+ (errmsg("verified existence of \"%s\" via basic_archive",
+ file)));
+ return true;
+}
diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index cc2f62bf36..8e36fa7ed6 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -31,4 +31,9 @@ tests += {
# typical installcheck users do not have (e.g. buildfarm clients).
'runningcheck': false,
},
+ 'tap': {
+ 'tests': [
+ 't/001_restore.pl',
+ ],
+ },
}
diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl
new file mode 100644
index 0000000000..6c01f736cf
--- /dev/null
+++ b/contrib/basic_archive/t/001_restore.pl
@@ -0,0 +1,44 @@
+
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# start a node
+my $node = PostgreSQL::Test::Cluster->new('node');
+$node->init(has_archiving => 1, allows_streaming => 1);
+my $archive_dir = $node->archive_dir;
+$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os;
+$node->append_conf('postgresql.conf', "archive_command = ''");
+$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'");
+$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$node->start;
+
+# backup the node
+my $backup = 'backup';
+$node->backup($backup);
+
+# generate some new WAL files
+$node->safe_psql('postgres', "CREATE TABLE test (a INT);");
+$node->safe_psql('postgres', "SELECT pg_switch_wal();");
+$node->safe_psql('postgres', "INSERT INTO test VALUES (1);");
+
+# shut down the node (this should archive all WAL files)
+$node->stop;
+
+# restore from the backup
+my $restore = PostgreSQL::Test::Cluster->new('restore');
+$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0);
+$restore->append_conf('postgresql.conf', "restore_command = ''");
+$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'");
+$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'");
+$restore->start;
+
+# ensure post-backup WAL is replayed
+my $result = $restore->poll_query_until("postgres", "SELECT count(*) = 1 FROM test;");
+is($result, "1", "check restore content");
+
+done_testing();
diff --git a/doc/src/sgml/archive-and-restore-modules.sgml b/doc/src/sgml/archive-and-restore-modules.sgml
index a52d15082d..15be548a73 100644
--- a/doc/src/sgml/archive-and-restore-modules.sgml
+++ b/doc/src/sgml/archive-and-restore-modules.sgml
@@ -8,15 +8,17 @@
<para>
PostgreSQL provides infrastructure to create custom modules for continuous
- archiving (see <xref linkend="continuous-archiving"/>). While archiving via
- a shell command (i.e., <xref linkend="guc-archive-command"/>) is much
- simpler, a custom archive module will often be considerably more robust and
- performant.
+ archiving and recovery (see <xref linkend="continuous-archiving"/>). While a
+ shell command (i.e., <xref linkend="guc-archive-command"/>,
+ <xref linkend="guc-restore-command"/>) is much simpler, a custom module will
+ often be considerably more robust and performant.
</para>
<para>
The <filename>contrib/basic_archive</filename> module contains a working
- example, which demonstrates some useful techniques.
+ example, which demonstrates some useful techniques. As demonstrated in
+ <filename>basic_archive</filename> a single module may serve as both an
+ archive module and a restore module.
</para>
<sect1 id="archive-modules">
@@ -178,4 +180,348 @@ typedef void (*ArchiveShutdownCB) (ArchiveModuleState *state);
</sect3>
</sect2>
</sect1>
+
+ <sect1 id="restore-modules">
+ <title>Restore Modules</title>
+ <indexterm zone="restore-modules">
+ <primary>Restore Modules</primary>
+ </indexterm>
+
+ <para>
+ When a custom <xref linkend="guc-restore-library"/> is configured,
+ PostgreSQL will use the module for restore actions. It is
+ ultimately up to the module to decide how to accomplish each task,
+ but some recommendations are listed at
+ <xref linkend="backup-pitr-recovery"/>.
+ </para>
+
+ <para>
+ Restore modules must at least consist of an initialization function
+ (see <xref linkend="restore-module-init"/>) and the required callbacks (see
+ <xref linkend="restore-module-callbacks"/>). However, restore modules are
+ also permitted to do much more (e.g., declare GUCs and register background
+ workers).
+ </para>
+
+ <sect2 id="restore-module-init">
+ <title>Initialization Functions</title>
+ <indexterm zone="restore-module-init">
+ <primary>_PG_restore_module_init</primary>
+ </indexterm>
+
+ <para>
+ An restore library is loaded by dynamically loading a shared library with
+ the <xref linkend="guc-restore-library"/>'s name as the library base name.
+ The normal library search path is used to locate the library. To provide
+ the required restore module callbacks and to indicate that the library is
+ actually a restore module, it needs to provide a function named
+ <function>_PG_restore_module_init</function>. The result of the function
+ must be a pointer to a struct of type
+ <structname>RestoreModuleCallbacks</structname>, which contains everything
+ that the core code needs to know how to make use of the restore module.
+ The return value needs to be of server lifetime, which is typically
+ achieved by defining it as a <literal>static const</literal> variable in
+ global scope.
+
+<programlisting>
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+</programlisting>
+
+ The <function>restore_wal_segment_configured_cb</function>,
+ <function>restore_wal_segment_cb</function>,
+ <function>restore_timeline_history_configured_cb</function>,
+ <function>restore_timeline_history_cb</function>,
+ <function>timeline_history_exists_configured_cb</function>, and
+ <function>timeline_history_exists_cb</function> callbacks are required for
+ archive recovery but optional for streaming replication. The others are
+ always optional.
+ </para>
+ </sect2>
+
+ <sect2 id="restore-module-callbacks">
+ <title>Restore Module Callbacks</title>
+
+ <para>
+ The restore callbacks define the actual behavior of the module. The server
+ will call them as required to execute restore actions.
+ </para>
+
+ <sect3 id="restore-module-startup">
+ <title>Startup Callback</title>
+
+ <para>
+ The <function>startup_cb</function> callback is called shortly after the
+ module is loaded. This callback can be used to perform any additional
+ initialization required. If the restore module has state, it can use
+ <structfield>state->private_data</structfield> to store it.
+
+<programlisting>
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+</programlisting>
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment-configured">
+ <title>Restore WAL Segment Configured Callback</title>
+ <para>
+ The <function>restore_wal_segment_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to retrieve WAL files (e.g., its configuration parameters
+ are set to valid values). If no
+ <function>restore_wal_segment_configured_cb</function> is defined, the
+ server always assumes the module is not configured to retrieve WAL files.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve WAL files
+ by calling the <function>restore_wal_segment_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_wal_segment_cb</function> callback to retrieve WAL
+ files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-wal-segment">
+ <title>Restore WAL Segment Callback</title>
+ <para>
+ The <function>restore_wal_segment_cb</function> callback is called to
+ retrieve a single archived segment of the WAL file series for archive
+ recovery or streaming replication.
+
+<programlisting>
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state, const char *file, const char *path, const char *lastRestartPointFileName);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point. That is the earliest
+ file that must be kept to allow a restore to be restartable, so this
+ information can be used to truncate the archive to just the minimum
+ required to support restarting from the current restore.
+ <replaceable>lastRestartPointFileName</replaceable> is typically only used
+ by warm-standby configurations (see <xref linkend="warm-standby"/>). Note
+ that if multiple standby servers are restoring from the same archive
+ directory, you will need to ensure that you do not delete WAL files until
+ they are no longer needed by any of the servers.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history-configured">
+ <title>Restore Timeline History Configured Callback</title>
+ <para>
+ The <function>restore_timeline_history_configured_cb</function> callback
+ is called to determine whether the module is fully configured and ready to
+ accept requests to retrieve timeline history files (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>restore_timeline_history_configured_cb</function> is defined,
+ the server always assumes the module is not configured to retrieve
+ timeline history files.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will retrieve timeline
+ history files by calling the
+ <function>restore_timeline_history_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>restore_timeline_history_cb</function> callback to retrieve
+ timeline history files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-restore-timeline-history">
+ <title>Restore Timeline History Callback</title>
+ <para>
+ The <function>restore_timeline_history_cb</function> callback is called to
+ retrieve a single archived timeline history file for archive recovery or
+ streaming replication.
+
+<programlisting>
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file was
+ successfully retrieved. If the file is not available in the archives, the
+ callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the WAL file to retrieve, while <replaceable>path</replaceable>
+ contains the destination's relative path (including the file name).
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists-configured">
+ <title>Timeline History Exists Configured Callback</title>
+ <para>
+ The <function>timeline_history_exists_configured_cb</function> callback is
+ called to determine whether the module is fully configured and ready to
+ accept requests to determine whether a timeline history file exists in the
+ archives (e.g., its configuration parameters are set to valid values). If
+ no <function>timeline_history_exists_configured_cb</function> is defined,
+ the server always assumes the module is not configured to check whether
+ certain timeline history files exist.
+
+<programlisting>
+typedef bool (*TimelineHistoryConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will check whether
+ certain timeline history files are present in the archives by calling the
+ <function>timeline_history_exists_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>timeline_history_exists_cb</function> callback to check if
+ timeline history files exist in the archives.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-timeline-history-exists">
+ <title>Timeline History Exists Callback</title>
+ <para>
+ The <function>timeline_history_exists_cb</function> callback is called to
+ check whether a timeline history file is present in the archives.
+
+<programlisting>
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state, const char *file, const char *path);
+</programlisting>
+
+ This callback must return <literal>true</literal> only if the file is
+ present in the archives. If the file is not available in the archives,
+ the callback must return <literal>false</literal>.
+ <replaceable>file</replaceable> will contain just the file name
+ of the timeline history file.
+ </para>
+
+ <para>
+ Some restore modules might not have a dedicated way to determine whether a
+ timeline history file exists. For example,
+ <varname>restore_command</varname> only tells the server how to retrieve
+ files. In this case, the <function>timeline_history_exists_cb</function>
+ callback should copy the file to <replaceable>path</replaceable>, which
+ contains the destination's relative path (including the file name), and
+ the restore module should set
+ <structfield>state->timeline_history_exists_cb_copies</structfield> to
+ <literal>true</literal>. It is recommended to set this flag in the
+ module's <function>startup_cb</function> callback. This flag tells the
+ server that it should verify that the file was successfully retrieved
+ after invoking this callback.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup-configured">
+ <title>Archive Cleanup Configured Callback</title>
+ <para>
+ The <function>archive_cleanup_configured_cb</function> callback is called
+ to determine whether the module is fully configured and ready to accept
+ requests to remove old WAL files from the archives (e.g., its
+ configuration parameters are set to valid values). If no
+ <function>archive_cleanup_configured_cb</function> is defined, the server
+ always assumes the module is not configured to remove old WAL files.
+
+<programlisting>
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will remove old WAL
+ files by calling the <function>archive_cleanup_cb</function> callback. If
+ <literal>false</literal> is returned, the server will not use the
+ <function>archive_cleanup_cb</function> callback to remove old WAL files.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-archive-cleanup">
+ <title>Archive Cleanup Callback</title>
+ <para>
+ The <function>archive_cleanup_cb</function> callback is called at every
+ restart point by the checkpointer process and is intended to provide a
+ mechanism for cleaning up old archived WAL files that are no longer
+ needed by the standby server.
+
+<programlisting>
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the
+ name of the file that includes the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end-configured">
+ <title>Recovery End Configured Callback</title>
+ <para>
+ The <function>recovery_end_configured_cb</function> callback is called to
+ determine whether the module is fully configured and ready to execute its
+ <function>recovery_end_cb</function> callback once at the end of recovery
+ (e.g., its configuration parameters are set to valid values). If no
+ <function>recovery_end_configured_cb</function> is defined, the server
+ always assumes the module is not configured to execute its
+ <function>recovery_end_cb</function> callback.
+
+<programlisting>
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+</programlisting>
+
+ If <literal>true</literal> is returned, the server will execute the
+ <function>recovery_end_cb</function> callback once at the end of recovery.
+ If <literal>false</literal> is returned, the server will not use the
+ <function>recovery_end_cb</function> callback at the end of recovery.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-recovery-end">
+ <title>Recovery End Callback</title>
+ <para>
+ The <function>recovery_end_cb</function> callback is called once at the
+ end of recovery and is intended to provide a mechanism for cleanup
+ following the end of replication or recovery.
+
+<programlisting>
+typedef void (*RecoveryEndCB) (RestoreModuleState *state, const char *lastRestartPointFileName);
+</programlisting>
+
+ <replaceable>lastRestartPointFileName</replaceable> will contain the name
+ of the file containing the last valid restart point, like in
+ <link linkend="restore-module-restore-wal-segment"><function>restore_wal_segment_cb</function></link>.
+ </para>
+ </sect3>
+
+ <sect3 id="restore-module-shutdown">
+ <title>Shutdown Callback</title>
+ <para>
+ The <function>shutdown_cb</function> callback is called when a process
+ that has loaded the restore module exits (e.g., after an error) or the
+ value of <xref linkend="guc-restore-library"/> changes. If no
+ <function>shutdown_cb</function> is defined, no special action is taken in
+ these situations. If the restore module has state, this callback should
+ free it to avoid leaks.
+
+<programlisting>
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+ </programlisting>
+ </para>
+ </sect3>
+ </sect2>
+ </sect1>
</chapter>
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index b3468eea3c..e1643f92e0 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1261,9 +1261,26 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
<para>
The key part of all this is to set up a recovery configuration that
describes how you want to recover and how far the recovery should
- run. The one thing that you absolutely must specify is the <varname>restore_command</varname>,
+ run. The one thing that you absolutely must specify is either
+ <varname>restore_command</varname> or a <varname>restore_library</varname>,
which tells <productname>PostgreSQL</productname> how to retrieve archived
- WAL file segments. Like the <varname>archive_command</varname>, this is
+ WAL file segments.
+ </para>
+
+ <para>
+ Like the <varname>archive_library</varname> parameter,
+ <varname>restore_library</varname> is a shared library. Since such
+ libraries are written in <literal>C</literal>, creating your own may
+ require considerably more effort than writing a shell command. However,
+ restore modules can be more performance than restoring via shell, and they
+ will have access to many useful server resources. For more information
+ about creating a <varname>restore_library</varname>, see
+ <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ Like the <varname>archive_command</varname>,
+ <varname>restore_command</varname> is
a shell command string. It can contain <literal>%f</literal>, which is
replaced by the name of the desired WAL file, and <literal>%p</literal>,
which is replaced by the path name to copy the WAL file to.
@@ -1282,14 +1299,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
</para>
<para>
- It is important that the command return nonzero exit status on failure.
- The command <emphasis>will</emphasis> be called requesting files that are not
- present in the archive; it must return nonzero when so asked. This is not
- an error condition. An exception is that if the command was terminated by
+ It is important that the <varname>restore_command</varname> return nonzero
+ exit status on failure, or, if you are using a
+ <varname>restore_library</varname>, that the restore callbacks return
+ <literal>false</literal> on failure. The command or library
+ <emphasis>will</emphasis> be called requesting files that are not
+ present in the archive; it must fail when so asked. This is not
+ an error condition. An exception is that if the
+ <varname>restore_command</varname> was terminated by
a signal (other than <systemitem>SIGTERM</systemitem>, which is used as
part of a database server shutdown) or an error by the shell (such as
command not found), then recovery will abort and the server will not start
- up.
+ up. Likewise, if the restore callbacks provided by the
+ <varname>restore_library</varname> emit an <literal>ERROR</literal> or
+ <literal>FATAL</literal>, recovery will abort and the server won't start.
</para>
<para>
@@ -1313,7 +1336,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
close as possible given the available WAL segments). Therefore, a normal
recovery will end with a <quote>file not found</quote> message, the exact text
of the error message depending upon your choice of
- <varname>restore_command</varname>. You may also see an error message
+ <varname>restore_command</varname> or <varname>restore_library</varname>.
+ You may also see an error message
at the start of recovery for a file named something like
<filename>00000001.history</filename>. This is also normal and does not
indicate a problem in simple recovery situations; see
diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml
index b4d43ced20..f3a5a502bd 100644
--- a/doc/src/sgml/basic-archive.sgml
+++ b/doc/src/sgml/basic-archive.sgml
@@ -8,17 +8,20 @@
</indexterm>
<para>
- <filename>basic_archive</filename> is an example of an archive module. This
- module copies completed WAL segment files to the specified directory. This
- may not be especially useful, but it can serve as a starting point for
- developing your own archive module. For more information about archive
- modules, see <xref linkend="archive-modules"/>.
+ <filename>basic_archive</filename> is an example of an archive and restore
+ module. This module copies completed WAL segment files to or from the
+ specified directory. This may not be especially useful, but it can serve as
+ a starting point for developing your own archive and restore modules. For
+ more information about archive and restore modules, see\
+ <xref linkend="archive-and-restore-modules"/>.
</para>
<para>
- In order to function, this module must be loaded via
+ For use as an archive module, this module must be loaded via
<xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/>
- must be enabled.
+ must be enabled. For use as a restore module, this module must be loaded
+ via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see
+ <xref linkend="runtime-config-wal-archive-recovery"/>).
</para>
<sect2 id="basic-archive-configuration-parameters">
@@ -34,11 +37,12 @@
</term>
<listitem>
<para>
- The directory where the server should copy WAL segment files. This
- directory must already exist. The default is an empty string, which
- effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/>
- is enabled, the server will accumulate WAL segment files in the
- expectation that a value will soon be provided.
+ The directory where the server should copy WAL segment files to or from.
+ This directory must already exist. The default is an empty string,
+ which, when used for archiving, effectively halts WAL archival, but if
+ <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate
+ WAL segment files in the expectation that a value will soon be provided.
+ When an empty string is used for recovery, restore will fail.
</para>
</listitem>
</varlistentry>
@@ -46,7 +50,7 @@
<para>
These parameters must be set in <filename>postgresql.conf</filename>.
- Typical usage might be:
+ Typical usage as an archive module might be:
</para>
<programlisting>
@@ -61,6 +65,7 @@ basic_archive.archive_directory = '/path/to/archive/directory'
<title>Notes</title>
<para>
+ When <filename>basic_archive</filename> is used as an archive module,
Server crashes may leave temporary files with the prefix
<filename>archtemp</filename> in the archive directory. It is recommended to
delete such files before restarting the server after a crash. It is safe to
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index b38cbd714a..bba79e19ab 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3906,7 +3906,8 @@ include_dir 'conf.d'
recovery when the end of archived WAL is reached, but will keep trying to
continue recovery by connecting to the sending server as specified by the
<varname>primary_conninfo</varname> setting and/or by fetching new WAL
- segments using <varname>restore_command</varname>. For this mode, the
+ segments using <varname>restore_command</varname> or
+ <varname>restore_library</varname>. For this mode, the
parameters from this section and <xref
linkend="runtime-config-replication-standby"/> are of interest.
Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will
@@ -3934,7 +3935,8 @@ include_dir 'conf.d'
<listitem>
<para>
The local shell command to execute to retrieve an archived segment of
- the WAL file series. This parameter is required for archive recovery,
+ the WAL file series. Either <varname>restore_command</varname> or
+ <xref linkend="guc-restore-library"/> is required for archive recovery,
but optional for streaming replication.
Any <literal>%f</literal> in the string is
replaced by the name of the file to retrieve from the archive,
@@ -3969,7 +3971,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>restore_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry id="guc-restore-library" xreflabel="restore_library">
+ <term><varname>restore_library</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>restore_library</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ The library to use for restore actions, including retrieving archived
+ files and executing tasks at restartpoints and at recovery end. Either
+ <xref linkend="guc-restore-command"/> or
+ <varname>restore_library</varname> is required for archive recovery,
+ but optional for streaming replication. If this parameter is set to an
+ empty string (the default), restoring via shell is enabled, and
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, and
+ <varname>recovery_end_command</varname> are used. If both
+ <varname>restore_library</varname> and any of
+ <varname>restore_command</varname>,
+ <varname>archive_cleanup_command</varname>, or
+ <varname>recovery_end_command</varname> are set, an error will be
+ raised. Otherwise, the specified shared library is used for restoring.
+ For more information, see <xref linkend="restore-modules"/>.
+ </para>
+
+ <para>
+ This parameter can only be set in the
+ <filename>postgresql.conf</filename> file or on the server command line.
</para>
</listitem>
</varlistentry>
@@ -4014,7 +4051,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>archive_cleanup_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
@@ -4043,7 +4083,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</para>
<para>
This parameter can only be set in the <filename>postgresql.conf</filename>
- file or on the server command line.
+ file or on the server command line. It is only used if
+ <varname>restore_library</varname> is set to an empty string. If both
+ <varname>recovery_end_command</varname> and
+ <varname>restore_library</varname> are set, an error will be raised.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml
index 236c0af65f..f811398fa2 100644
--- a/doc/src/sgml/high-availability.sgml
+++ b/doc/src/sgml/high-availability.sgml
@@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order.
<para>
In standby mode, the server continuously applies WAL received from the
primary server. The standby server can read WAL from a WAL archive
- (see <xref linkend="guc-restore-command"/>) or directly from the primary
+ (see <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/>) or directly from the primary
over a TCP connection (streaming replication). The standby server will
also attempt to restore any WAL found in the standby cluster's
<filename>pg_wal</filename> directory. That typically happens after a server
@@ -638,8 +639,10 @@ protocol to make nodes agree on a serializable transactional order.
<para>
At startup, the standby begins by restoring all WAL available in the
- archive location, calling <varname>restore_command</varname>. Once it
+ archive location, either by calling <varname>restore_command</varname> or
+ by executing the <varname>restore_library</varname>'s callbacks. Once it
reaches the end of WAL available there and <varname>restore_command</varname>
+ or <varname>restore_library</varname>
fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory.
If that fails, and streaming replication has been configured, the
standby tries to connect to the primary server and start streaming WAL
@@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order.
server (see <xref linkend="backup-pitr-recovery"/>). Create a file
<link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm>
in the standby's cluster data
- directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from
+ directory. Set <xref linkend="guc-restore-command"/> or
+ <xref linkend="guc-restore-library"/> to copy files from
the WAL archive. If you plan to have multiple standby servers for high
availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to
<literal>latest</literal> (the default), to make the standby server follow the timeline change
@@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order.
<note>
<para>
- <xref linkend="guc-restore-command"/> should return immediately
+ <xref linkend="guc-restore-command"/> and
+ <xref linkend="guc-restore-library"/> should return immediately
if the file does not exist; the server will retry the command again if
necessary.
</para>
@@ -731,8 +736,9 @@ protocol to make nodes agree on a serializable transactional order.
<para>
If you're using a WAL archive, its size can be minimized using the <xref
- linkend="guc-archive-cleanup-command"/> parameter to remove files that are no
- longer required by the standby server.
+ linkend="guc-archive-cleanup-command"/> parameter or the
+ <xref linkend="guc-restore-library"/>'s archive cleanup callbacks to remove
+ files that are no longer required by the standby server.
The <application>pg_archivecleanup</application> utility is designed specifically to
be used with <varname>archive_cleanup_command</varname> in typical single-standby
configurations, see <xref linkend="pgarchivecleanup"/>.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e5d12f1d15..34a21c40a5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -83,7 +83,7 @@
#include "replication/snapbuild.h"
#include "replication/walreceiver.h"
#include "replication/walsender.h"
-#include "restore/shell_restore.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
@@ -5178,18 +5178,19 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog,
TimeLineID newTLI)
{
/*
- * Execute the recovery_end_command, if any.
+ * Execute the recovery-end callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_recovery_end_configured())
+ if (recovery_end_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_recovery_end(lastRestartPointFname);
+ RestoreCallbacks->recovery_end_cb(restore_module_state,
+ lastRestartPointFname);
}
/*
@@ -7673,18 +7674,19 @@ CreateRestartPoint(int flags)
timestamptz_to_str(xtime)) : 0));
/*
- * Finally, execute archive_cleanup_command, if any.
+ * Finally, execute archive-cleanup callback, if any.
*
- * The command is provided the archive file cutoff point for use during
+ * The callback is provided the archive file cutoff point for use during
* log shipping replication. All files earlier than this point can be
* deleted from the archive, though there is no requirement to do so.
*/
- if (shell_archive_cleanup_configured())
+ if (archive_cleanup_configured())
{
char lastRestartPointFname[MAXFNAMELEN];
GetOldestRestartPointFileName(lastRestartPointFname);
- shell_archive_cleanup(lastRestartPointFname);
+ RestoreCallbacks->archive_cleanup_cb(restore_module_state,
+ lastRestartPointFname);
}
return true;
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 320fa857ea..8ebbd6ce50 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -23,14 +23,21 @@
#include "access/xlog_internal.h"
#include "access/xlogarchive.h"
#include "common/archive.h"
+#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "postmaster/startup.h"
#include "postmaster/pgarch.h"
#include "replication/walsender.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/fd.h"
#include "storage/ipc.h"
+#include "utils/memutils.h"
+
+char *restoreLibrary = "";
+const RestoreModuleCallbacks *RestoreCallbacks = NULL;
+RestoreModuleState *restore_module_state;
/*
* Attempt to retrieve the specified file from off-line archival storage.
@@ -74,15 +81,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- if (!shell_restore_file_configured())
+ if (!restore_wal_segment_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- if (!shell_restore_file_configured())
+ if (!restore_timeline_history_configured())
goto not_available;
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- if (!shell_restore_file_configured())
+ if (!timeline_history_exists_configured())
goto not_available;
break;
}
@@ -174,14 +181,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
switch (archive_type)
{
case ARCHIVE_TYPE_WAL_SEGMENT:
- ret = shell_restore_wal_segment(xlogfname, xlogpath,
- lastRestartPointFname);
+ ret = RestoreCallbacks->restore_wal_segment_cb(restore_module_state,
+ xlogfname, xlogpath,
+ lastRestartPointFname);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->restore_timeline_history_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS:
- ret = shell_restore_timeline_history(xlogfname, xlogpath);
+ ret = RestoreCallbacks->timeline_history_exists_cb(restore_module_state,
+ xlogfname, xlogpath);
break;
}
@@ -189,6 +199,16 @@ RestoreArchivedFile(char *path, const char *xlogfname,
if (ret)
{
+ /*
+ * Some restore functions might not copy the file (e.g., checking
+ * whether a timeline history file exists), but they can set a flag to
+ * tell us if they do. We only need to verify file existence if this
+ * flag is enabled.
+ */
+ if (archive_type == ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS &&
+ !restore_module_state->timeline_history_exists_cb_copies)
+ return true;
+
/*
* command apparently succeeded, but let's make sure the file is
* really there now and has the correct size.
@@ -630,3 +650,65 @@ XLogArchiveCleanup(const char *xlog)
unlink(archiveStatusPath);
/* should we complain about failure? */
}
+
+/*
+ * Loads all the restore callbacks into our global RestoreCallbacks. The
+ * caller is responsible for validating the combination of library/command
+ * parameters that are set (e.g., restore_command and restore_library cannot
+ * both be set).
+ */
+void
+LoadRestoreCallbacks(void)
+{
+ RestoreModuleInit init;
+
+ /*
+ * If the shell command is enabled, use our special initialization
+ * function. Otherwise, load the library and call its
+ * _PG_restore_module_init().
+ */
+ if (restoreLibrary[0] == '\0')
+ init = shell_restore_init;
+ else
+ init = (RestoreModuleInit)
+ load_external_function(restoreLibrary, "_PG_restore_module_init",
+ false, NULL);
+
+ if (init == NULL)
+ ereport(ERROR,
+ (errmsg("restore modules have to define the symbol "
+ "_PG_restore_module_init")));
+
+ RestoreCallbacks = (*init) ();
+
+ /* restore state should be freed before calling this function */
+ Assert(restore_module_state == NULL);
+ restore_module_state = (RestoreModuleState *)
+ MemoryContextAllocZero(TopMemoryContext,
+ sizeof(RestoreModuleState));
+
+ if (RestoreCallbacks->startup_cb != NULL)
+ RestoreCallbacks->startup_cb(restore_module_state);
+}
+
+/*
+ * Call the shutdown callback of the loaded restore module, if defined. Also,
+ * free the restore module state if it was allocated.
+ *
+ * Processes that load restore libraries should register this via
+ * before_shmem_exit().
+ */
+void
+call_restore_module_shutdown_cb(int code, Datum arg)
+{
+ if (RestoreCallbacks != NULL &&
+ RestoreCallbacks->shutdown_cb != NULL &&
+ restore_module_state != NULL)
+ RestoreCallbacks->shutdown_cb(restore_module_state);
+
+ if (restore_module_state != NULL)
+ {
+ pfree(restore_module_state);
+ restore_module_state = NULL;
+ }
+}
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 0bfa569502..4afdcc3aad 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -51,6 +51,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/walreceiver.h"
+#include "restore/restore_module.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/latch.h"
@@ -1114,18 +1115,21 @@ validateRecoveryParameters(void)
if (StandbyModeRequested)
{
if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) &&
- (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0))
+ (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured()))
ereport(WARNING,
- (errmsg("specified neither primary_conninfo nor restore_command"),
+ (errmsg("specified neither primary_conninfo nor restore_command nor a configured restore_library"),
errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there.")));
}
else
{
- if (recoveryRestoreCommand == NULL ||
- strcmp(recoveryRestoreCommand, "") == 0)
+ if (!restore_wal_segment_configured() ||
+ !restore_timeline_history_configured() ||
+ !timeline_history_exists_configured())
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("must specify restore_command when standby mode is not enabled")));
+ errmsg("must specify restore_command or a configured restore_library when standby mode is not enabled")));
}
/*
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 46197d56f8..47993aa35e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -45,6 +45,7 @@
#include "postmaster/bgwriter.h"
#include "postmaster/interrupt.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/fd.h"
@@ -224,6 +225,25 @@ CheckpointerMain(void)
ALLOCSET_DEFAULT_SIZES);
MemoryContextSwitchTo(checkpointer_context);
+ /*
+ * Load restore_library so that we can use its archive-cleanup callback,
+ * if one is defined.
+ *
+ * We also take this opportunity to set up the before_shmem_exit hook for
+ * the shutdown callback and to check that only one of restore_library,
+ * archive_cleanup_command is set. Note that we emit a WARNING upon
+ * detecting misconfigured parameters, and we clear the callbacks so that
+ * the archive-cleanup functionality is skipped. We judge this
+ * functionality to not be important enough to require blocking
+ * checkpoints or shutting down the server when the parameters are
+ * misconfigured.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING))
+ LoadRestoreCallbacks();
+
/*
* If an exception is encountered, processing resumes here.
*
@@ -556,6 +576,10 @@ HandleCheckpointerInterrupts(void)
if (ConfigReloadPending)
{
+ bool archive_cleanup_settings_changed = false;
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand);
+
ConfigReloadPending = false;
ProcessConfigFile(PGC_SIGHUP);
@@ -571,6 +595,36 @@ HandleCheckpointerInterrupts(void)
* because of SIGHUP.
*/
UpdateSharedMemoryConfig();
+
+ /*
+ * If the archive-cleanup settings have changed, call the currently
+ * loaded shutdown callback and clear all the restore callbacks.
+ * There's presently no good way to unload a library besides
+ * restarting the process, and there should be little harm in leaving
+ * it around, so we just leave it loaded.
+ */
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0)
+ {
+ archive_cleanup_settings_changed = true;
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ RestoreCallbacks = NULL;
+ }
+
+ /*
+ * As in CheckpointerMain(), we only emit a WARNING if we detect that
+ * both restore_library and archive_cleanup_command are set. We do
+ * this even if the archive-cleanup settings haven't changed to remind
+ * the user about the misconfiguration.
+ */
+ if (CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ archiveCleanupCommand, "archive_cleanup_command",
+ WARNING) &&
+ archive_cleanup_settings_changed)
+ LoadRestoreCallbacks();
+
+ pfree(prevRestoreLibrary);
+ pfree(prevArchiveCleanupCommand);
}
if (ShutdownRequestPending)
{
diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c
index e391bf6aa6..3cbd382e00 100644
--- a/src/backend/postmaster/startup.c
+++ b/src/backend/postmaster/startup.c
@@ -25,6 +25,7 @@
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "storage/ipc.h"
#include "storage/pmsignal.h"
#include "storage/procsignal.h"
@@ -118,13 +119,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS)
* Re-read the config file.
*
* If one of the critical walreceiver options has changed, flag xlog.c
- * to restart it.
+ * to restart it. Also, check for invalid combinations of the command/library
+ * parameters and reload the restore callbacks if necessary.
*/
static void
StartupRereadConfig(void)
{
char *conninfo = pstrdup(PrimaryConnInfo);
char *slotname = pstrdup(PrimarySlotName);
+ char *prevRestoreLibrary = pstrdup(restoreLibrary);
+ char *prevRestoreCommand = pstrdup(recoveryRestoreCommand);
+ char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand);
bool tempSlot = wal_receiver_create_temp_slot;
bool conninfoChanged;
bool slotnameChanged;
@@ -146,6 +151,30 @@ StartupRereadConfig(void)
if (conninfoChanged || slotnameChanged || tempSlotChanged)
StartupRequestWalReceiverRestart();
+
+ /*
+ * If the restore settings have changed, call the currently loaded
+ * shutdown callback and load the new callbacks. There's presently no
+ * good way to unload a library besides restarting the process, and there
+ * should be little harm in leaving it around, so we just leave it loaded.
+ */
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 ||
+ strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 ||
+ strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0)
+ {
+ call_restore_module_shutdown_cb(0, (Datum) 0);
+ LoadRestoreCallbacks();
+ }
+
+ pfree(prevRestoreLibrary);
+ pfree(prevRestoreCommand);
+ pfree(prevRecoveryEndCommand);
}
/* Handle various signals that might be sent to the startup process */
@@ -255,6 +284,19 @@ StartupProcessMain(void)
*/
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
+ /*
+ * Check for invalid combinations of the command/library parameters and
+ * load the callbacks.
+ */
+ before_shmem_exit(call_restore_module_shutdown_cb, 0);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryRestoreCommand, "restore_command",
+ ERROR);
+ (void) CheckMutuallyExclusiveStringGUCs(restoreLibrary, "restore_library",
+ recoveryEndCommand, "recovery_end_command",
+ ERROR);
+ LoadRestoreCallbacks();
+
/*
* Do what we came for.
*/
diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c
index 42291625c1..be25f04044 100644
--- a/src/backend/restore/shell_restore.c
+++ b/src/backend/restore/shell_restore.c
@@ -3,7 +3,8 @@
* shell_restore.c
*
* These restore functions use a user-specified shell command (e.g., the
- * restore_command GUC).
+ * restore_command GUC). It is used as the default, but other modules may
+ * define their own restore logic.
*
* Copyright (c) 2024, PostgreSQL Global Development Group
*
@@ -22,25 +23,76 @@
#include "common/archive.h"
#include "common/percentrepl.h"
#include "postmaster/startup.h"
+#include "restore/restore_module.h"
#include "restore/shell_restore.h"
#include "storage/ipc.h"
#include "utils/wait_event.h"
+static void shell_restore_startup(RestoreModuleState *state);
+static bool shell_restore_file_configured(RestoreModuleState *state);
static bool shell_restore_file(const char *file, const char *path,
const char *lastRestartPointFileName);
+static bool shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+static bool shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path);
+static bool shell_archive_cleanup_configured(RestoreModuleState *state);
+static void shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+static bool shell_recovery_end_configured(RestoreModuleState *state);
+static void shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName);
static void ExecuteRecoveryCommand(const char *command,
const char *commandName,
bool failOnSignal,
uint32 wait_event_info,
const char *lastRestartPointFileName);
+static const RestoreModuleCallbacks shell_restore_callbacks = {
+ .startup_cb = shell_restore_startup,
+ .restore_wal_segment_configured_cb = shell_restore_file_configured,
+ .restore_wal_segment_cb = shell_restore_wal_segment,
+ .restore_timeline_history_configured_cb = shell_restore_file_configured,
+ .restore_timeline_history_cb = shell_restore_timeline_history,
+ .timeline_history_exists_configured_cb = shell_restore_file_configured,
+ .timeline_history_exists_cb = shell_restore_timeline_history,
+ .archive_cleanup_configured_cb = shell_archive_cleanup_configured,
+ .archive_cleanup_cb = shell_archive_cleanup,
+ .recovery_end_configured_cb = shell_recovery_end_configured,
+ .recovery_end_cb = shell_recovery_end,
+ .shutdown_cb = NULL
+};
+
+/*
+ * shell_restore_init
+ *
+ * Returns the callbacks for restoring via shell.
+ */
+const RestoreModuleCallbacks *
+shell_restore_init(void)
+{
+ return &shell_restore_callbacks;
+}
+
+/*
+ * Indicates that our timeline_history_exists_cb only attempts to retrieve the
+ * file, so the caller should verify the file exists afterwards.
+ */
+static void
+shell_restore_startup(RestoreModuleState *state)
+{
+ state->timeline_history_exists_cb_copies = true;
+}
+
/*
* Attempt to execute a shell-based restore command to retrieve a WAL segment.
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_wal_segment(const char *file, const char *path,
+static bool
+shell_restore_wal_segment(RestoreModuleState *state,
+ const char *file, const char *path,
const char *lastRestartPointFileName)
{
return shell_restore_file(file, path, lastRestartPointFileName);
@@ -52,8 +104,9 @@ shell_restore_wal_segment(const char *file, const char *path,
*
* Returns true if the command has succeeded, false otherwise.
*/
-bool
-shell_restore_timeline_history(const char *file, const char *path)
+static bool
+shell_restore_timeline_history(RestoreModuleState *state,
+ const char *file, const char *path)
{
char lastRestartPointFname[MAXPGPATH];
@@ -64,8 +117,8 @@ shell_restore_timeline_history(const char *file, const char *path)
/*
* Check whether restore_command is supplied.
*/
-bool
-shell_restore_file_configured(void)
+static bool
+shell_restore_file_configured(RestoreModuleState *state)
{
return recoveryRestoreCommand[0] != '\0';
}
@@ -152,8 +205,8 @@ shell_restore_file(const char *file, const char *path,
/*
* Check whether archive_cleanup_command is supplied.
*/
-bool
-shell_archive_cleanup_configured(void)
+static bool
+shell_archive_cleanup_configured(RestoreModuleState *state)
{
return archiveCleanupCommand[0] != '\0';
}
@@ -161,8 +214,9 @@ shell_archive_cleanup_configured(void)
/*
* Attempt to execute a shell-based archive cleanup command.
*/
-void
-shell_archive_cleanup(const char *lastRestartPointFileName)
+static void
+shell_archive_cleanup(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command",
false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND,
@@ -172,8 +226,8 @@ shell_archive_cleanup(const char *lastRestartPointFileName)
/*
* Check whether recovery_end_command is supplied.
*/
-bool
-shell_recovery_end_configured(void)
+static bool
+shell_recovery_end_configured(RestoreModuleState *state)
{
return recoveryEndCommand[0] != '\0';
}
@@ -181,8 +235,9 @@ shell_recovery_end_configured(void)
/*
* Attempt to execute a shell-based end-of-recovery command.
*/
-void
-shell_recovery_end(const char *lastRestartPointFileName)
+static void
+shell_recovery_end(RestoreModuleState *state,
+ const char *lastRestartPointFileName)
{
ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true,
WAIT_EVENT_RECOVERY_END_COMMAND,
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 45013582a7..724d268d7b 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -70,6 +70,7 @@
#include "replication/slot.h"
#include "replication/slotsync.h"
#include "replication/syncrep.h"
+#include "restore/restore_module.h"
#include "storage/bufmgr.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -3946,6 +3947,16 @@ struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
+ gettext_noop("Sets the library that will be called for restore actions."),
+ NULL
+ },
+ &restoreLibrary,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY,
gettext_noop("Sets the shell command that will be executed at every restart point."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index edcc0282b2..2939042b06 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -284,6 +284,7 @@
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
+#restore_library = '' # library to use for restore actions
#archive_cleanup_command = '' # command to execute at every restartpoint
#recovery_end_command = '' # command to execute at completion of recovery
diff --git a/src/include/restore/restore_module.h b/src/include/restore/restore_module.h
new file mode 100644
index 0000000000..cc148e855a
--- /dev/null
+++ b/src/include/restore/restore_module.h
@@ -0,0 +1,143 @@
+/*-------------------------------------------------------------------------
+ *
+ * restore_module.h
+ * Exports for restore modules.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * src/include/restore/restore_module.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _RESTORE_MODULE_H
+#define _RESTORE_MODULE_H
+
+/*
+ * The value of the restore_library GUC.
+ */
+extern PGDLLIMPORT char *restoreLibrary;
+
+typedef struct RestoreModuleState
+{
+ /*
+ * Private data pointer for use by a restore module. This can be used to
+ * store state for the module that will be passed to each of its
+ * callbacks.
+ */
+ void *private_data;
+
+ /*
+ * Indicates whether the callback that checks for timeline existence
+ * merely copies the file. If set to true, the server will verify that
+ * the timeline history file exists after the callback returns.
+ */
+ bool timeline_history_exists_cb_copies;
+} RestoreModuleState;
+
+/*
+ * Restore module callbacks
+ *
+ * These callback functions should be defined by restore libraries and returned
+ * via _PG_restore_module_init(). For more information about the purpose of
+ * each callback, refer to the restore modules documentation.
+ */
+typedef void (*RestoreStartupCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreWalSegmentCB) (RestoreModuleState *state,
+ const char *file, const char *path,
+ const char *lastRestartPointFileName);
+typedef bool (*RestoreTimelineHistoryConfiguredCB) (RestoreModuleState *state);
+typedef bool (*RestoreTimelineHistoryCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*TimelineHistoryExistsConfiguredCB) (RestoreModuleState *state);
+typedef bool (*TimelineHistoryExistsCB) (RestoreModuleState *state,
+ const char *file, const char *path);
+typedef bool (*ArchiveCleanupConfiguredCB) (RestoreModuleState *state);
+typedef void (*ArchiveCleanupCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef bool (*RecoveryEndConfiguredCB) (RestoreModuleState *state);
+typedef void (*RecoveryEndCB) (RestoreModuleState *state,
+ const char *lastRestartPointFileName);
+typedef void (*RestoreShutdownCB) (RestoreModuleState *state);
+
+typedef struct RestoreModuleCallbacks
+{
+ RestoreStartupCB startup_cb;
+ RestoreWalSegmentConfiguredCB restore_wal_segment_configured_cb;
+ RestoreWalSegmentCB restore_wal_segment_cb;
+ RestoreTimelineHistoryConfiguredCB restore_timeline_history_configured_cb;
+ RestoreTimelineHistoryCB restore_timeline_history_cb;
+ TimelineHistoryExistsConfiguredCB timeline_history_exists_configured_cb;
+ TimelineHistoryExistsCB timeline_history_exists_cb;
+ ArchiveCleanupConfiguredCB archive_cleanup_configured_cb;
+ ArchiveCleanupCB archive_cleanup_cb;
+ RecoveryEndConfiguredCB recovery_end_configured_cb;
+ RecoveryEndCB recovery_end_cb;
+ RestoreShutdownCB shutdown_cb;
+} RestoreModuleCallbacks;
+
+/*
+ * Type of the shared library symbol _PG_restore_module_init that is looked up
+ * when loading a restore library.
+ */
+typedef const RestoreModuleCallbacks *(*RestoreModuleInit) (void);
+
+extern PGDLLEXPORT const RestoreModuleCallbacks *_PG_restore_module_init(void);
+
+extern void LoadRestoreCallbacks(void);
+extern void call_restore_module_shutdown_cb(int code, Datum arg);
+
+extern const RestoreModuleCallbacks *RestoreCallbacks;
+extern RestoreModuleState *restore_module_state;
+
+/*
+ * The following *_configured() functions are convenient wrappers around the
+ * *_configured_cb() callback functions with additional defensive NULL checks.
+ */
+
+static inline bool
+restore_wal_segment_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_cb != NULL &&
+ RestoreCallbacks->restore_wal_segment_configured_cb(restore_module_state);
+}
+
+static inline bool
+restore_timeline_history_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_cb != NULL &&
+ RestoreCallbacks->restore_timeline_history_configured_cb(restore_module_state);
+}
+
+static inline bool
+timeline_history_exists_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_cb != NULL &&
+ RestoreCallbacks->timeline_history_exists_configured_cb(restore_module_state);
+}
+
+static inline bool
+archive_cleanup_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_cb != NULL &&
+ RestoreCallbacks->archive_cleanup_configured_cb(restore_module_state);
+}
+
+static inline bool
+recovery_end_configured(void)
+{
+ return RestoreCallbacks != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb != NULL &&
+ RestoreCallbacks->recovery_end_cb != NULL &&
+ RestoreCallbacks->recovery_end_configured_cb(restore_module_state);
+}
+
+#endif /* _RESTORE_MODULE_H */
diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h
index 28b5b7bc92..6352623866 100644
--- a/src/include/restore/shell_restore.h
+++ b/src/include/restore/shell_restore.h
@@ -12,15 +12,13 @@
#ifndef _SHELL_RESTORE_H
#define _SHELL_RESTORE_H
-extern bool shell_restore_file_configured(void);
-extern bool shell_restore_wal_segment(const char *file, const char *path,
- const char *lastRestartPointFileName);
-extern bool shell_restore_timeline_history(const char *file, const char *path);
+#include "restore/restore_module.h"
-extern bool shell_archive_cleanup_configured(void);
-extern void shell_archive_cleanup(const char *lastRestartPointFileName);
-
-extern bool shell_recovery_end_configured(void);
-extern void shell_recovery_end(const char *lastRestartPointFileName);
+/*
+ * Since the logic for restoring via shell commands is in the core server and
+ * does not need to be loaded via a shared library, it has a special
+ * initialization function.
+ */
+extern const RestoreModuleCallbacks *shell_restore_init(void);
#endif /* _SHELL_RESTORE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 3a02310f26..6a05313f6b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2386,6 +2386,8 @@ ResourceOwnerDesc
ResourceReleaseCallback
ResourceReleaseCallbackItem
ResourceReleasePhase
+RestoreModuleCallbacks
+RestoreModuleState
RestoreOptions
RestorePass
RestrictInfo
--
2.25.1
--IJpNTDwzlM2Ie8A6--
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
@ 2024-01-15 06:27 Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-15 06:27 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
If there are no more comments for the current design, I'll
start implementing this feature with the following
approaches for "Discussing" items:
> 3.1 Should we use one function(internal) for COPY TO/FROM
> handlers or two function(internal)s (one is for COPY TO
> handler and another is for COPY FROM handler)?
> [4]
I'll choose "one function(internal) for COPY TO/FROM handlers".
> 3.4 Should we export Copy{To,From}State? Or should we just
> provide getters/setters to access Copy{To,From}State
> internal?
> [10]
I'll export Copy{To,From}State.
Thanks,
--
kou
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST),
Sutou Kouhei <[email protected]> wrote:
> Hi,
>
> Here is the current summary for a this discussion to make
> COPY format extendable. It's for reaching consensus and
> starting implementing the feature. (I'll start implementing
> the feature once we reach consensus.) If you have any
> opinion, please share it.
>
> Confirmed:
>
> 1.1 Making COPY format extendable will not reduce performance.
> [1]
>
> Decisions:
>
> 2.1 Use separated handler for COPY TO and COPY FROM because
> our COPY TO implementation (copyto.c) and COPY FROM
> implementation (coypfrom.c) are separated.
> [2]
>
> 2.2 Don't use system catalog for COPY TO/FROM handlers. We can
> just use a function(internal) that returns a handler instead.
> [3]
>
> 2.3 The implementation must include documentation.
> [5]
>
> 2.4 The implementation must include test.
> [6]
>
> 2.5 The implementation should be consist of small patches
> for easy to review.
> [6]
>
> 2.7 Copy{To,From}State must have a opaque space for
> handlers.
> [8]
>
> 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO
> handlers.
> [8]
>
> 2.9 Make "format" in PgMsg_CopyOutResponse message
> extendable.
> [9]
>
> 2.10 Make makeNode() call avoidable in function(internal)
> that returns COPY TO/FROM handler.
> [9]
>
> 2.11 Custom COPY TO/FROM handlers must be able to parse
> their options.
> [11]
>
> Discussing:
>
> 3.1 Should we use one function(internal) for COPY TO/FROM
> handlers or two function(internal)s (one is for COPY TO
> handler and another is for COPY FROM handler)?
> [4]
>
> 3.2 If we use separated function(internal) for COPY TO/FROM
> handlers, we need to define naming rule. For example,
> <method_name>_to(internal) for COPY TO handler and
> <method_name>_from(internal) for COPY FROM handler.
> [4]
>
> 3.3 Should we use prefix or suffix for function(internal)
> name to avoid name conflict with other handlers such as
> tablesample handlers?
> [7]
>
> 3.4 Should we export Copy{To,From}State? Or should we just
> provide getters/setters to access Copy{To,From}State
> internal?
> [10]
>
>
> [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com
> [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz
> [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m...
> [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m...
> [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp...
> [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz
> [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com
> [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com
> [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz
> [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m...
> [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com
>
>
> Thanks,
> --
> kou
>
>
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-24 05:49 ` Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 14:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 2 replies; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-24 05:49 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
I've implemented custom COPY format feature based on the
current design discussion. See the attached patches for
details.
I also implemented a PoC COPY format handler for Apache
Arrow with this implementation and it worked.
https://github.com/kou/pg-copy-arrow
The patches implement not only custom COPY TO format feature
but also custom COPY FROM format feature.
0001-0004 is for COPY TO and 0005-0008 is for COPY FROM.
For COPY TO:
0001: This adds CopyToRoutine and use it for text/csv/binary
formats. No implementation change. This just move codes.
0002: This adds support for adding custom COPY TO format by
"CREATE FUNCTION ${FORMAT_NAME}". This uses the same
approach provided by Sawada-san[1] but this doesn't
introduce a wrapper CopyRoutine struct for
Copy{To,From}Routine. Because I noticed that a wrapper
CopyRoutine struct is needless. Copy handler can just return
CopyToRoutine or CopyFromRtouine because both of them have
NodeTag. We can distinct a returned struct by the NodeTag.
[1] https://www.postgresql.org/message-id/[email protected]...
0003: This exports CopyToStateData. No implementation change
except CopyDest enum values. I changed COPY_ prefix to
COPY_DEST_ to avoid name conflict with CopySource enum
values. This just moves codes.
0004: This adds CopyToState::opaque and exports
CopySendEndOfRow(). CopySendEndOfRow() is renamed to
CopyToStateFlush().
For COPY FROM:
0005: Same as 0001 but for COPY FROM. This adds
CopyFromRoutine and use it for text/csv/binary formats. No
implementation change. This just move codes.
0006: Same as 0002 but for COPY FROM. This adds support for
adding custom COPY FROM format by "CREATE FUNCTION
${FORMAT_NAME}".
0007: Same as 0003 but for COPY FROM. This exports
CopyFromStateData. No implementation change except
CopySource enum values. I changed COPY_ prefix to
COPY_SOURCE_ to align CopyDest changes in 0003. This just
moves codes.
0008: Same as 0004 but for COPY FROM. This adds
CopyFromState::opaque and exports
CopyReadBinaryData(). CopyReadBinaryData() is renamed to
CopyFromStateRead().
Thanks,
--
kou
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 15 Jan 2024 15:27:02 +0900 (JST),
Sutou Kouhei <[email protected]> wrote:
> Hi,
>
> If there are no more comments for the current design, I'll
> start implementing this feature with the following
> approaches for "Discussing" items:
>
>> 3.1 Should we use one function(internal) for COPY TO/FROM
>> handlers or two function(internal)s (one is for COPY TO
>> handler and another is for COPY FROM handler)?
>> [4]
>
> I'll choose "one function(internal) for COPY TO/FROM handlers".
>
>> 3.4 Should we export Copy{To,From}State? Or should we just
>> provide getters/setters to access Copy{To,From}State
>> internal?
>> [10]
>
> I'll export Copy{To,From}State.
>
>
> Thanks,
> --
> kou
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 12 Jan 2024 14:46:15 +0900 (JST),
> Sutou Kouhei <[email protected]> wrote:
>
>> Hi,
>>
>> Here is the current summary for a this discussion to make
>> COPY format extendable. It's for reaching consensus and
>> starting implementing the feature. (I'll start implementing
>> the feature once we reach consensus.) If you have any
>> opinion, please share it.
>>
>> Confirmed:
>>
>> 1.1 Making COPY format extendable will not reduce performance.
>> [1]
>>
>> Decisions:
>>
>> 2.1 Use separated handler for COPY TO and COPY FROM because
>> our COPY TO implementation (copyto.c) and COPY FROM
>> implementation (coypfrom.c) are separated.
>> [2]
>>
>> 2.2 Don't use system catalog for COPY TO/FROM handlers. We can
>> just use a function(internal) that returns a handler instead.
>> [3]
>>
>> 2.3 The implementation must include documentation.
>> [5]
>>
>> 2.4 The implementation must include test.
>> [6]
>>
>> 2.5 The implementation should be consist of small patches
>> for easy to review.
>> [6]
>>
>> 2.7 Copy{To,From}State must have a opaque space for
>> handlers.
>> [8]
>>
>> 2.8 Export CopySendData() and CopySendEndOfRow() for COPY TO
>> handlers.
>> [8]
>>
>> 2.9 Make "format" in PgMsg_CopyOutResponse message
>> extendable.
>> [9]
>>
>> 2.10 Make makeNode() call avoidable in function(internal)
>> that returns COPY TO/FROM handler.
>> [9]
>>
>> 2.11 Custom COPY TO/FROM handlers must be able to parse
>> their options.
>> [11]
>>
>> Discussing:
>>
>> 3.1 Should we use one function(internal) for COPY TO/FROM
>> handlers or two function(internal)s (one is for COPY TO
>> handler and another is for COPY FROM handler)?
>> [4]
>>
>> 3.2 If we use separated function(internal) for COPY TO/FROM
>> handlers, we need to define naming rule. For example,
>> <method_name>_to(internal) for COPY TO handler and
>> <method_name>_from(internal) for COPY FROM handler.
>> [4]
>>
>> 3.3 Should we use prefix or suffix for function(internal)
>> name to avoid name conflict with other handlers such as
>> tablesample handlers?
>> [7]
>>
>> 3.4 Should we export Copy{To,From}State? Or should we just
>> provide getters/setters to access Copy{To,From}State
>> internal?
>> [10]
>>
>>
>> [1] https://www.postgresql.org/message-id/flat/20231204.153548.2126325458835528809.kou%40clear-code.com
>> [2] https://www.postgresql.org/message-id/flat/ZXEUIy6wl4jHy6Nm%40paquier.xyz
>> [3] https://www.postgresql.org/message-id/flat/CAD21AoAhcZkAp_WDJ4sSv_%2Bg2iCGjfyMFgeu7MxjnjX_FutZAg%40m...
>> [4] https://www.postgresql.org/message-id/flat/CAD21AoDkoGL6yJ_HjNOg9cU%3DaAdW8uQ3rSQOeRS0SX85LPPNwQ%40m...
>> [5] https://www.postgresql.org/message-id/flat/TY3PR01MB9889C9234CD220A3A7075F0DF589A%40TY3PR01MB9889.jp...
>> [6] https://www.postgresql.org/message-id/flat/ZXbiPNriHHyUrcTF%40paquier.xyz
>> [7] https://www.postgresql.org/message-id/flat/20231214.184414.2179134502876898942.kou%40clear-code.com
>> [8] https://www.postgresql.org/message-id/flat/20231221.183504.1240642084042888377.kou%40clear-code.com
>> [9] https://www.postgresql.org/message-id/flat/ZYTfqGppMc9e_w2k%40paquier.xyz
>> [10] https://www.postgresql.org/message-id/flat/CAD21AoD%3DUapH4Wh06G6H5XAzPJ0iJg9YcW8r7E2UEJkZ8QsosA%40m...
>> [11] https://www.postgresql.org/message-id/flat/20240110.152023.1920937326588672387.kou%40clear-code.com
>>
>>
>> Thanks,
>> --
>> kou
>>
>>
>
>
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-24 08:11 ` Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
1 sibling, 1 reply; 51+ messages in thread
From: Michael Paquier @ 2024-01-24 08:11 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
> For COPY TO:
>
> 0001: This adds CopyToRoutine and use it for text/csv/binary
> formats. No implementation change. This just move codes.
10M without this change:
format,elapsed time (ms)
text,1090.763
csv,1136.103
binary,1137.141
10M with this change:
format,elapsed time (ms)
text,1082.654
csv,1196.991
binary,1069.697
These numbers point out that binary is faster by 6%, csv is slower by
5%, while text stays around what looks like noise range. That's not
negligible. Are these numbers reproducible? If they are, that could
be a problem for anybody doing bulk-loading of large data sets. I am
not sure to understand where the improvement for binary comes from by
reading the patch, but perhaps perf would tell more for each format?
The loss with csv could be blamed on the extra manipulations of the
function pointers, likely.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-24 12:15 ` Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Andrew Dunstan @ 2024-01-24 12:15 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; pgsql-hackers
On 2024-01-24 We 03:11, Michael Paquier wrote:
> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
>> For COPY TO:
>>
>> 0001: This adds CopyToRoutine and use it for text/csv/binary
>> formats. No implementation change. This just move codes.
> 10M without this change:
>
> format,elapsed time (ms)
> text,1090.763
> csv,1136.103
> binary,1137.141
>
> 10M with this change:
>
> format,elapsed time (ms)
> text,1082.654
> csv,1196.991
> binary,1069.697
>
> These numbers point out that binary is faster by 6%, csv is slower by
> 5%, while text stays around what looks like noise range. That's not
> negligible. Are these numbers reproducible? If they are, that could
> be a problem for anybody doing bulk-loading of large data sets. I am
> not sure to understand where the improvement for binary comes from by
> reading the patch, but perhaps perf would tell more for each format?
> The loss with csv could be blamed on the extra manipulations of the
> function pointers, likely.
I don't think that's at all acceptable.
We've spent quite a lot of blood sweat and tears over the years to make
COPY fast, and we should not sacrifice any of that lightly.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
@ 2024-01-24 14:17 ` Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 3 replies; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-24 14:17 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500,
Andrew Dunstan <[email protected]> wrote:
>
> On 2024-01-24 We 03:11, Michael Paquier wrote:
>> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
>>> For COPY TO:
>>>
>>> 0001: This adds CopyToRoutine and use it for text/csv/binary
>>> formats. No implementation change. This just move codes.
>> 10M without this change:
>>
>> format,elapsed time (ms)
>> text,1090.763
>> csv,1136.103
>> binary,1137.141
>>
>> 10M with this change:
>>
>> format,elapsed time (ms)
>> text,1082.654
>> csv,1196.991
>> binary,1069.697
>>
>> These numbers point out that binary is faster by 6%, csv is slower by
>> 5%, while text stays around what looks like noise range. That's not
>> negligible. Are these numbers reproducible? If they are, that could
>> be a problem for anybody doing bulk-loading of large data sets. I am
>> not sure to understand where the improvement for binary comes from by
>> reading the patch, but perhaps perf would tell more for each format?
>> The loss with csv could be blamed on the extra manipulations of the
>> function pointers, likely.
>
>
> I don't think that's at all acceptable.
>
> We've spent quite a lot of blood sweat and tears over the years to make COPY
> fast, and we should not sacrifice any of that lightly.
These numbers aren't reproducible. Because these benchmarks
executed on my normal machine not a machine only for
benchmarking. The machine runs another processes such as
editor and Web browser.
For example, here are some results with master
(94edfe250c6a200d2067b0debfe00b4122e9b11e):
Format,N records,Elapsed time (ms)
csv,10000000,1073.715
csv,10000000,1022.830
csv,10000000,1073.584
csv,10000000,1090.651
csv,10000000,1052.259
Here are some results with master + the 0001 patch:
Format,N records,Elapsed time (ms)
csv,10000000,1025.356
csv,10000000,1067.202
csv,10000000,1014.563
csv,10000000,1032.088
csv,10000000,1058.110
I uploaded my benchmark script so that you can run the same
benchmark on your machine:
https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
Could anyone try the benchmark with master and master+0001?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 02:53 ` jian he <[email protected]>
2024-01-25 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 2 replies; 51+ messages in thread
From: jian he @ 2024-01-25 02:53 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 10:17 PM Sutou Kouhei <[email protected]> wrote:
>
> I uploaded my benchmark script so that you can run the same
> benchmark on your machine:
>
> https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
>
> Could anyone try the benchmark with master and master+0001?
>
sorry. I made a mistake. I applied v6, 0001 to 0008 all the patches.
my tests:
CREATE unlogged TABLE data (a bigint);
SELECT setseed(0.29);
INSERT INTO data SELECT random() * 10000 FROM generate_series(1, 1e7);
my setup:
meson setup --reconfigure ${BUILD} \
-Dprefix=${PG_PREFIX} \
-Dpgport=5462 \
-Dbuildtype=release \
-Ddocs_html_style=website \
-Ddocs_pdf=disabled \
-Dllvm=disabled \
-Dextra_version=_release_build
gcc version: PostgreSQL 17devel_release_build on x86_64-linux,
compiled by gcc-11.4.0, 64-bit
apply your patch:
COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
Time: 668.996 ms
Time: 596.254 ms
Time: 592.723 ms
Time: 591.663 ms
Time: 590.803 ms
not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda
COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
Time: 644.246 ms
Time: 583.075 ms
Time: 568.670 ms
Time: 569.463 ms
Time: 569.201 ms
I forgot to test other formats.
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
@ 2024-01-25 03:28 ` Michael Paquier <[email protected]>
1 sibling, 0 replies; 51+ messages in thread
From: Michael Paquier @ 2024-01-25 03:28 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 10:53:58AM +0800, jian he wrote:
> apply your patch:
> COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
> Time: 668.996 ms
> Time: 596.254 ms
> Time: 592.723 ms
> Time: 591.663 ms
> Time: 590.803 ms
>
> not apply your patch, at git 729439607ad210dbb446e31754e8627d7e3f7dda
> COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
> Time: 644.246 ms
> Time: 583.075 ms
> Time: 568.670 ms
> Time: 569.463 ms
> Time: 569.201 ms
>
> I forgot to test other formats.
There can be some variance in the tests, so you'd better run much more
tests so as you can get a better idea of the mean. Discarding the N
highest and lowest values also reduces slightly the effects of the
noise you would get across single runs.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
@ 2024-01-25 08:05 ` Sutou Kouhei <[email protected]>
1 sibling, 0 replies; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-25 08:05 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
Thanks for trying these patches!
In <CACJufxF9NS3xQ2d79jN0V1CGvF7cR16uJo-C3nrY7vZrwvxF7w@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 10:53:58 +0800,
jian he <[email protected]> wrote:
> COPY data TO '/dev/null' WITH (FORMAT csv) \watch count=5
Wow! I didn't know the "\watch count="!
I'll use it.
> Time: 668.996 ms
> Time: 596.254 ms
> Time: 592.723 ms
> Time: 591.663 ms
> Time: 590.803 ms
It seems that 5 times isn't enough for this case as Michael
said. But thanks for trying!
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 03:17 ` Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 1 reply; 51+ messages in thread
From: Michael Paquier @ 2024-01-25 03:17 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 11:17:26PM +0900, Sutou Kouhei wrote:
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500,
> Andrew Dunstan <[email protected]> wrote:
>> We've spent quite a lot of blood sweat and tears over the years to make COPY
>> fast, and we should not sacrifice any of that lightly.
Clearly.
> I uploaded my benchmark script so that you can run the same
> benchmark on your machine:
>
> https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
Thanks, that saves time. I am attaching it to this email as well, for
the sake of the archives if this link is removed in the future.
> Could anyone try the benchmark with master and master+0001?
Yep. It is one point we need to settle before deciding what to do
with this patch set, and I've done so to reach my own conclusion.
I have a rather good machine at my disposal in the cloud, so I did a
few runs with HEAD and HEAD+0001, with PGDATA mounted on a tmpfs.
Here are some results for the 10M row case, as these should be the
least prone to noise, 5 runs each:
master
text 10M 1732.570 1684.542 1693.430 1687.696 1714.845
csv 10M 1729.113 1724.926 1727.414 1726.237 1728.865
bin 10M 1679.097 1677.887 1676.764 1677.554 1678.120
master+0001
text 10M 1702.207 1654.818 1647.069 1690.568 1654.446
csv 10M 1764.939 1714.313 1712.444 1712.323 1716.952
bin 10M 1703.061 1702.719 1702.234 1703.346 1704.137
Hmm. The point of contention in the patch is the change to use the
CopyToOneRow callback in CopyOneRowTo(), as we go through it for each
row and we should habe a worst-case scenario with a relation that has
a small attribute size. The more rows, the more effect it would have.
The memory context switches and the StringInfo manipulations are
equally important, and there are a bunch of the latter, actually, with
optimizations around fe_msgbuf.
I've repeated a few runs across these two builds, and there is some
variance and noise, but I am going to agree with your point that the
effect 0001 cannot be seen. Even HEAD is showing some noise. So I am
discarding the concerns I had after seeing the numbers you posted
upthread.
+typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
+typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
+typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
+typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
+typedef void (*CopyToEnd_function) (CopyToState cstate);
We don't really need a set of typedefs here, let's put the definitions
in the CopyToRoutine struct instead.
+extern CopyToRoutine CopyToRoutineText;
+extern CopyToRoutine CopyToRoutineCSV;
+extern CopyToRoutine CopyToRoutineBinary;
All that should IMO remain in copyto.c and copyfrom.c in the initial
patch doing the refactoring. Why not using a fetch function instead
that uses a string in input? Then you can call that once after
parsing the List of options in ProcessCopyOptions().
Introducing copyapi.h in the initial patch makes sense here for the TO
and FROM routines.
+/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
+ * move the code to here later. */
Some areas, like this comment, are written in an incorrect format.
+ if (cstate->opts.csv_mode)
+ CopyAttributeOutCSV(cstate, colname, false,
+ list_length(cstate->attnumlist) == 1);
+ else
+ CopyAttributeOutText(cstate, colname);
You are right that this is not worth the trouble of creating a
different set of callbacks for CSV. This makes the result cleaner.
+ getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
Actually, this split is interesting. It is possible for a custom
format to plug in a custom set of out functions. Did you make use of
something custom for your own stuff? Actually, could it make sense to
split the assignment of cstate->out_functions into its own callback?
Sure, that's part of the start phase, but at least it would make clear
that a custom method *has* to assign these OIDs to work. The patch
implies that as a rule, without a comment that CopyToStart *must* set
up these OIDs.
I think that 0001 and 0005 should be handled first, as pieces
independent of the rest. Then we could move on with 0002~0004 and
0006~0008.
--
Michael
#!/bin/bash
set -eu
set -o pipefail
base_dir=$(dirname "$0")
prepare_sql()
{
size=$1
db_name=$2
cat <<SQL
DROP DATABASE IF EXISTS ${db_name};
CREATE DATABASE ${db_name};
\\c ${db_name}
CREATE TABLE data (int32 integer);
SELECT setseed(0.29);
INSERT INTO data
SELECT random() * 10000
FROM generate_series(1, ${size});
SQL
}
measure()
{
local format=$1
local n_tries=6
for i in $(seq ${n_tries}); do
LANG=C \
PAGER=cat \
psql \
--no-psqlrc \
--command "\\timing" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \
grep --text -o '^Time: [0-9.]* ms' | \
grep -o '[0-9.]*'
done | sort --numeric-sort | head -n 9 | tail -n 1
}
result_suffix=""
if [ $# -gt 0 ]; then
result_suffix="-$1"
fi
result="${base_dir}/result${result_suffix}.csv"
echo "Format,N records,Elapsed time (ms)" | \
tee "${result}"
sizes=()
sizes+=(100000)
sizes+=(1000000)
sizes+=(10000000)
for size in "${sizes[@]}"; do
export PGDATABASE="copy_${size}"
echo "${size}: preparing"
prepare_sql "${size}" "${PGDATABASE}" | \
psql -d postgres
echo "${size}: text"
elapsed_time=$(measure text)
echo "text,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: csv"
elapsed_time=$(measure csv)
echo "csv,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: binary"
elapsed_time=$(measure binary)
echo "binary,${size},${elapsed_time}" | \
tee -a "${result}"
done
Attachments:
[text/plain] bench-run.txt (1.6K, ../../[email protected]/2-bench-run.txt)
download | inline:
#!/bin/bash
set -eu
set -o pipefail
base_dir=$(dirname "$0")
prepare_sql()
{
size=$1
db_name=$2
cat <<SQL
DROP DATABASE IF EXISTS ${db_name};
CREATE DATABASE ${db_name};
\\c ${db_name}
CREATE TABLE data (int32 integer);
SELECT setseed(0.29);
INSERT INTO data
SELECT random() * 10000
FROM generate_series(1, ${size});
SQL
}
measure()
{
local format=$1
local n_tries=6
for i in $(seq ${n_tries}); do
LANG=C \
PAGER=cat \
psql \
--no-psqlrc \
--command "\\timing" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" \
--command "COPY data TO '/dev/null' WITH (FORMAT ${format})" | \
grep --text -o '^Time: [0-9.]* ms' | \
grep -o '[0-9.]*'
done | sort --numeric-sort | head -n 9 | tail -n 1
}
result_suffix=""
if [ $# -gt 0 ]; then
result_suffix="-$1"
fi
result="${base_dir}/result${result_suffix}.csv"
echo "Format,N records,Elapsed time (ms)" | \
tee "${result}"
sizes=()
sizes+=(100000)
sizes+=(1000000)
sizes+=(10000000)
for size in "${sizes[@]}"; do
export PGDATABASE="copy_${size}"
echo "${size}: preparing"
prepare_sql "${size}" "${PGDATABASE}" | \
psql -d postgres
echo "${size}: text"
elapsed_time=$(measure text)
echo "text,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: csv"
elapsed_time=$(measure csv)
echo "csv,${size},${elapsed_time}" | \
tee -a "${result}"
echo "${size}: binary"
elapsed_time=$(measure binary)
echo "binary,${size},${elapsed_time}" | \
tee -a "${result}"
done
[application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-25 08:45 ` Sutou Kouhei <[email protected]>
2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-25 08:45 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900,
Michael Paquier <[email protected]> wrote:
> +typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
> +typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
> +typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
> +typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
> +typedef void (*CopyToEnd_function) (CopyToState cstate);
>
> We don't really need a set of typedefs here, let's put the definitions
> in the CopyToRoutine struct instead.
OK. I'll do it.
> +extern CopyToRoutine CopyToRoutineText;
> +extern CopyToRoutine CopyToRoutineCSV;
> +extern CopyToRoutine CopyToRoutineBinary;
>
> All that should IMO remain in copyto.c and copyfrom.c in the initial
> patch doing the refactoring. Why not using a fetch function instead
> that uses a string in input? Then you can call that once after
> parsing the List of options in ProcessCopyOptions().
OK. How about the following for the fetch function
signature?
extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format);
We may introduce an enum and use it:
typedef enum CopyBuiltinFormat
{
COPY_BUILTIN_FORMAT_TEXT = 0,
COPY_BUILTIN_FORMAT_CSV,
COPY_BUILTIN_FORMAT_BINARY,
} CopyBuiltinFormat;
extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format);
> +/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
> + * move the code to here later. */
> Some areas, like this comment, are written in an incorrect format.
Oh, sorry. I assumed that the comment style was adjusted by
pgindent.
I'll use the following style:
/*
* ...
*/
> + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
>
> Actually, this split is interesting. It is possible for a custom
> format to plug in a custom set of out functions. Did you make use of
> something custom for your own stuff?
I didn't. My PoC custom COPY format handler for Apache Arrow
just handles integer and text for now. It doesn't use
cstate->out_functions because cstate->out_functions may not
return a valid binary format value for Apache Arrow. So it
formats each value by itself.
I'll chose one of them for a custom type (that isn't
supported by Apache Arrow, e.g. PostGIS types):
1. Report an unsupported error
2. Call output function for Apache Arrow provided by the
custom type
> Actually, could it make sense to
> split the assignment of cstate->out_functions into its own callback?
Yes. Because we need to use getTypeBinaryOutputInfo() for
"binary" and use getTypeOutputInfo() for "text" and "csv".
> Sure, that's part of the start phase, but at least it would make clear
> that a custom method *has* to assign these OIDs to work. The patch
> implies that as a rule, without a comment that CopyToStart *must* set
> up these OIDs.
CopyToStart doesn't need to set up them if the handler
doesn't use cstate->out_functions.
> I think that 0001 and 0005 should be handled first, as pieces
> independent of the rest. Then we could move on with 0002~0004 and
> 0006~0008.
OK. I'll focus on 0001 and 0005 for now. I'll restart
0002-0004/0006-0008 after 0001 and 0005 are accepted.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 23:35 ` Michael Paquier <[email protected]>
2024-01-26 08:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Michael Paquier @ 2024-01-25 23:35 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 05:45:43PM +0900, Sutou Kouhei wrote:
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 12:17:55 +0900,
> Michael Paquier <[email protected]> wrote:
>> +extern CopyToRoutine CopyToRoutineText;
>> +extern CopyToRoutine CopyToRoutineCSV;
>> +extern CopyToRoutine CopyToRoutineBinary;
>>
>> All that should IMO remain in copyto.c and copyfrom.c in the initial
>> patch doing the refactoring. Why not using a fetch function instead
>> that uses a string in input? Then you can call that once after
>> parsing the List of options in ProcessCopyOptions().
>
> OK. How about the following for the fetch function
> signature?
>
> extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format);
Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad
history with naming things around here.
> We may introduce an enum and use it:
>
> typedef enum CopyBuiltinFormat
> {
> COPY_BUILTIN_FORMAT_TEXT = 0,
> COPY_BUILTIN_FORMAT_CSV,
> COPY_BUILTIN_FORMAT_BINARY,
> } CopyBuiltinFormat;
>
> extern CopyToRoutine *GetBuiltinCopyToRoutine(CopyBuiltinFormat format);
I am not sure that this is necessary as the option value is a string.
> Oh, sorry. I assumed that the comment style was adjusted by
> pgindent.
No worries, that's just something we get used to. I tend to fix a lot
of these things by myself when editing patches.
>> + getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
>> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
>>
>> Actually, this split is interesting. It is possible for a custom
>> format to plug in a custom set of out functions. Did you make use of
>> something custom for your own stuff?
>
> I didn't. My PoC custom COPY format handler for Apache Arrow
> just handles integer and text for now. It doesn't use
> cstate->out_functions because cstate->out_functions may not
> return a valid binary format value for Apache Arrow. So it
> formats each value by itself.
I mean, if you use a custom output function, you could tweak things
even more with byteas or such.. If a callback is expected to do
something, like setting the output function OIDs in the start
callback, we'd better document it rather than letting that be implied.
>> Actually, could it make sense to
>> split the assignment of cstate->out_functions into its own callback?
>
> Yes. Because we need to use getTypeBinaryOutputInfo() for
> "binary" and use getTypeOutputInfo() for "text" and "csv".
Okay. After sleeping on it, a split makes sense here, because it also
reduces the presence of TupleDesc in the start callback.
>> Sure, that's part of the start phase, but at least it would make clear
>> that a custom method *has* to assign these OIDs to work. The patch
>> implies that as a rule, without a comment that CopyToStart *must* set
>> up these OIDs.
>
> CopyToStart doesn't need to set up them if the handler
> doesn't use cstate->out_functions.
Noted.
>> I think that 0001 and 0005 should be handled first, as pieces
>> independent of the rest. Then we could move on with 0002~0004 and
>> 0006~0008.
>
> OK. I'll focus on 0001 and 0005 for now. I'll restart
> 0002-0004/0006-0008 after 0001 and 0005 are accepted.
Once you get these, I'd be interested in re-doing an evaluation of
COPY TO and more tests with COPY FROM while running Postgres on
scissors. One thing I was thinking to use here is my blackhole_am for
COPY FROM:
https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am
As per its name, it does nothing on INSERT, so you could create a
table using it as access method, and stress the COPY FROM execution
paths without having to mount Postgres on a tmpfs because the data is
sent to the void. Perhaps it does not matter, but that moves the
tests to the bottlenecks we want to stress (aka the per-row callback
for large data sets).
I've switched the patch as waiting on author for now. Thanks for your
perseverance here. I understand that's not easy to follow up with
patches and reviews (^_^;)
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-26 08:49 ` Sutou Kouhei <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-26 08:49 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 08:35:19 +0900,
Michael Paquier <[email protected]> wrote:
>> OK. How about the following for the fetch function
>> signature?
>>
>> extern CopyToRoutine *GetBuiltinCopyToRoutine(const char *format);
>
> Or CopyToRoutineGet()? I am not wedded to my suggestion, got a bad
> history with naming things around here.
Thanks for the suggestion.
I rethink about this and use the following:
+extern void ProcessCopyOptionFormatTo(ParseState *pstate, CopyFormatOptions *opts_out, DefElem *defel);
It's not a fetch function. It sets CopyToRoutine opts_out
instead. But it hides CopyToRoutine* to copyto.c. Is it
acceptable?
>> OK. I'll focus on 0001 and 0005 for now. I'll restart
>> 0002-0004/0006-0008 after 0001 and 0005 are accepted.
>
> Once you get these, I'd be interested in re-doing an evaluation of
> COPY TO and more tests with COPY FROM while running Postgres on
> scissors. One thing I was thinking to use here is my blackhole_am for
> COPY FROM:
> https://github.com/michaelpq/pg_plugins/tree/main/blackhole_am
Thanks!
Could you evaluate the attached patch set with COPY FROM?
I attach v7 patch set. It includes only the 0001 and 0005
parts in v6 patch set because we focus on them for now.
0001: This is based on 0001 in v6.
Changes since v6:
* Fix comment style
* Hide CopyToRoutine{Text,CSV,Binary}
* Add more comments
* Eliminate "if (cstate->opts.csv_mode)" branches from "text"
and "csv" callbacks
* Remove CopyTo*_function typedefs
* Update benchmark results in commit message but the results
are measured on my environment that isn't suitable for
accurate benchmark
0002: This is based on 0005 in v6.
Changes since v6:
* Fix comment style
* Hide CopyFromRoutine{Text,CSV,Binary}
* Add more comments
* Eliminate a "if (cstate->opts.csv_mode)" branch from "text"
and "csv" callbacks
* NOTE: We can eliminate more "if (cstate->opts.csv_mode)" branches
such as one in NextCopyFromRawFields(). Should we do it
in this feature improvement (make COPY format
extendable)? Can we defer this as a separated improvement?
* Remove CopyFrom*_function typedefs
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-25 04:36 ` Masahiko Sawada <[email protected]>
2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2 siblings, 2 replies; 51+ messages in thread
From: Masahiko Sawada @ 2024-01-25 04:36 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 24, 2024 at 11:17 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <[email protected]>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 07:15:55 -0500,
> Andrew Dunstan <[email protected]> wrote:
>
> >
> > On 2024-01-24 We 03:11, Michael Paquier wrote:
> >> On Wed, Jan 24, 2024 at 02:49:36PM +0900, Sutou Kouhei wrote:
> >>> For COPY TO:
> >>>
> >>> 0001: This adds CopyToRoutine and use it for text/csv/binary
> >>> formats. No implementation change. This just move codes.
> >> 10M without this change:
> >>
> >> format,elapsed time (ms)
> >> text,1090.763
> >> csv,1136.103
> >> binary,1137.141
> >>
> >> 10M with this change:
> >>
> >> format,elapsed time (ms)
> >> text,1082.654
> >> csv,1196.991
> >> binary,1069.697
> >>
> >> These numbers point out that binary is faster by 6%, csv is slower by
> >> 5%, while text stays around what looks like noise range. That's not
> >> negligible. Are these numbers reproducible? If they are, that could
> >> be a problem for anybody doing bulk-loading of large data sets. I am
> >> not sure to understand where the improvement for binary comes from by
> >> reading the patch, but perhaps perf would tell more for each format?
> >> The loss with csv could be blamed on the extra manipulations of the
> >> function pointers, likely.
> >
> >
> > I don't think that's at all acceptable.
> >
> > We've spent quite a lot of blood sweat and tears over the years to make COPY
> > fast, and we should not sacrifice any of that lightly.
>
> These numbers aren't reproducible. Because these benchmarks
> executed on my normal machine not a machine only for
> benchmarking. The machine runs another processes such as
> editor and Web browser.
>
> For example, here are some results with master
> (94edfe250c6a200d2067b0debfe00b4122e9b11e):
>
> Format,N records,Elapsed time (ms)
> csv,10000000,1073.715
> csv,10000000,1022.830
> csv,10000000,1073.584
> csv,10000000,1090.651
> csv,10000000,1052.259
>
> Here are some results with master + the 0001 patch:
>
> Format,N records,Elapsed time (ms)
> csv,10000000,1025.356
> csv,10000000,1067.202
> csv,10000000,1014.563
> csv,10000000,1032.088
> csv,10000000,1058.110
>
>
> I uploaded my benchmark script so that you can run the same
> benchmark on your machine:
>
> https://gist.github.com/kou/be02e02e5072c91969469dbf137b5de5
>
> Could anyone try the benchmark with master and master+0001?
>
I've run a similar scenario:
create unlogged table test (a int);
insert into test select c from generate_series(1, 25000000) c;
copy test to '/tmp/result.csv' with (format csv); -- generates 230MB file
I've run it on HEAD and HEAD+0001 patch and here are the medians of 10
executions for each format:
HEAD:
binary 2930.353 ms
text 2754.852 ms
csv 2890.012 ms
HEAD w/ 0001 patch:
binary 2814.838 ms
text 2900.845 ms
csv 3015.210 ms
Hmm I can see a similar trend that Suto-san had; the binary format got
slightly faster whereas both text and csv format has small regression
(4%~5%). I think that the improvement for binary came from the fact
that we removed "if (cstate->opts.binary)" branches from the original
CopyOneRowTo(). I've experimented with a similar optimization for csv
and text format; have different callbacks for text and csv format and
remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
for that. Here are results:
HEAD w/ 0001 patch + remove branches:
binary 2824.502 ms
text 2715.264 ms
csv 2803.381 ms
The numbers look better now. I'm not sure these are within a noise
range but it might be worth considering having different callbacks for
text and csv formats.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] add_callback_for_csv_format.patch (3.9K, ../../CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com/2-add_callback_for_csv_format.patch)
download | inline diff:
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6547b7c654..f18b7d0823 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -223,11 +223,7 @@ CopyToTextStart(CopyToState cstate, TupleDesc tupDesc)
colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, colname, false,
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, colname);
+ CopyAttributeOutText(cstate, colname);
}
CopyToTextSendEndOfRow(cstate);
@@ -260,12 +256,7 @@ CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot)
char *string;
string = OutputFunctionCall(&out_functions[attnum - 1], value);
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, string,
- cstate->opts.force_quote_flags[attnum - 1],
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, string);
+ CopyAttributeOutText(cstate, string);
}
}
@@ -277,6 +268,99 @@ CopyToTextEnd(CopyToState cstate)
{
}
+static void
+CopyToCSVStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ int num_phys_attrs;
+ ListCell *cur;
+
+ num_phys_attrs = tupDesc->natts;
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
+
+ /*
+ * For non-binary copy, we need to convert null_print to file encoding,
+ * because it will be sent directly with CopySendString.
+ */
+ if (cstate->need_transcoding)
+ cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
+ cstate->opts.null_print_len,
+ cstate->file_encoding);
+
+ /* if a header has been requested send the line */
+ if (cstate->opts.header_line)
+ {
+ bool hdr_delim = false;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ char *colname;
+
+ if (hdr_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ hdr_delim = true;
+
+ colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
+
+ CopyAttributeOutCSV(cstate, colname, false,
+ list_length(cstate->attnumlist) == 1);
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+ }
+}
+
+static void
+CopyToCSVOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ bool need_delim = false;
+ FmgrInfo *out_functions = cstate->out_functions;
+ ListCell *cur;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Datum value = slot->tts_values[attnum - 1];
+ bool isnull = slot->tts_isnull[attnum - 1];
+
+ if (need_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ need_delim = true;
+
+ if (isnull)
+ {
+ CopySendString(cstate, cstate->opts.null_print_client);
+ }
+ else
+ {
+ char *string;
+
+ string = OutputFunctionCall(&out_functions[attnum - 1], value);
+ CopyAttributeOutCSV(cstate, string,
+ cstate->opts.force_quote_flags[attnum - 1],
+ list_length(cstate->attnumlist) == 1);
+ }
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+}
+
+static void
+CopyToCSVEnd(CopyToState cstate)
+{
+}
+
/*
* CopyToRoutine implementation for "binary".
*/
@@ -388,9 +472,9 @@ CopyToRoutine CopyToRoutineText = {
CopyToRoutine CopyToRoutineCSV = {
.CopyToProcessOption = CopyToTextProcessOption,
.CopyToGetFormat = CopyToTextGetFormat,
- .CopyToStart = CopyToTextStart,
- .CopyToOneRow = CopyToTextOneRow,
- .CopyToEnd = CopyToTextEnd,
+ .CopyToStart = CopyToCSVStart,
+ .CopyToOneRow = CopyToCSVOneRow,
+ .CopyToEnd = CopyToCSVEnd,
};
CopyToRoutine CopyToRoutineBinary = {
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-25 04:53 ` Michael Paquier <[email protected]>
2024-01-25 05:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
1 sibling, 1 reply; 51+ messages in thread
From: Michael Paquier @ 2024-01-25 04:53 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote:
> Hmm I can see a similar trend that Suto-san had; the binary format got
> slightly faster whereas both text and csv format has small regression
> (4%~5%). I think that the improvement for binary came from the fact
> that we removed "if (cstate->opts.binary)" branches from the original
> CopyOneRowTo(). I've experimented with a similar optimization for csv
> and text format; have different callbacks for text and csv format and
> remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> for that. Here are results:
>
> HEAD w/ 0001 patch + remove branches:
> binary 2824.502 ms
> text 2715.264 ms
> csv 2803.381 ms
>
> The numbers look better now. I'm not sure these are within a noise
> range but it might be worth considering having different callbacks for
> text and csv formats.
Interesting.
Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and
0.9% speedup for binary, which may be around the noise range assuming
a ~1% range. While this does not imply a regression, that seems worth
the duplication IMO. The patch had better document the reason why the
split is done, as well.
CopyFromTextOneRow() has also specific branches for binary and
non-binary removed in 0005, so assuming that I/O is not a bottleneck,
the operation would be faster because we would not evaluate this "if"
condition for each row. Wouldn't we also see improvements for COPY
FROM with short row values, say when mounting PGDATA into a
tmpfs/ramfs?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-25 05:28 ` Masahiko Sawada <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Masahiko Sawada @ 2024-01-25 05:28 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 1:53 PM Michael Paquier <[email protected]> wrote:
>
> On Thu, Jan 25, 2024 at 01:36:03PM +0900, Masahiko Sawada wrote:
> > Hmm I can see a similar trend that Suto-san had; the binary format got
> > slightly faster whereas both text and csv format has small regression
> > (4%~5%). I think that the improvement for binary came from the fact
> > that we removed "if (cstate->opts.binary)" branches from the original
> > CopyOneRowTo(). I've experimented with a similar optimization for csv
> > and text format; have different callbacks for text and csv format and
> > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> > for that. Here are results:
> >
> > HEAD w/ 0001 patch + remove branches:
> > binary 2824.502 ms
> > text 2715.264 ms
> > csv 2803.381 ms
> >
> > The numbers look better now. I'm not sure these are within a noise
> > range but it might be worth considering having different callbacks for
> > text and csv formats.
>
> Interesting.
>
> Your numbers imply a 0.3% speedup for text, 0.7% speedup for csv and
> 0.9% speedup for binary, which may be around the noise range assuming
> a ~1% range. While this does not imply a regression, that seems worth
> the duplication IMO.
Agreed. In addition to that, now that each format routine has its own
callbacks, there would be chances that we can do other optimizations
dedicated to the format type in the future if available.
> The patch had better document the reason why the
> split is done, as well.
+1
>
> CopyFromTextOneRow() has also specific branches for binary and
> non-binary removed in 0005, so assuming that I/O is not a bottleneck,
> the operation would be faster because we would not evaluate this "if"
> condition for each row. Wouldn't we also see improvements for COPY
> FROM with short row values, say when mounting PGDATA into a
> tmpfs/ramfs?
Probably. Seems worth evaluating.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-25 08:52 ` Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
1 sibling, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-25 08:52 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900,
Masahiko Sawada <[email protected]> wrote:
> I've experimented with a similar optimization for csv
> and text format; have different callbacks for text and csv format and
> remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> for that. Here are results:
>
> HEAD w/ 0001 patch + remove branches:
> binary 2824.502 ms
> text 2715.264 ms
> csv 2803.381 ms
>
> The numbers look better now. I'm not sure these are within a noise
> range but it might be worth considering having different callbacks for
> text and csv formats.
Wow! Interesting. I tried the approach before but I didn't
see any difference by the approach. But it may depend on my
environment.
I'll import the approach to the next patch set so that
others can try the approach easily.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-26 08:18 ` Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Junwang Zhao @ 2024-01-26 08:18 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Thu, Jan 25, 2024 at 4:52 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAD21AoALxEZz33NpcSk99ad_DT3A2oFNMa2KNjGBCMVFeCiUaA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Thu, 25 Jan 2024 13:36:03 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
> > I've experimented with a similar optimization for csv
> > and text format; have different callbacks for text and csv format and
> > remove "if (cstate->opts.csv_mode)" branches. I've attached a patch
> > for that. Here are results:
> >
> > HEAD w/ 0001 patch + remove branches:
> > binary 2824.502 ms
> > text 2715.264 ms
> > csv 2803.381 ms
> >
> > The numbers look better now. I'm not sure these are within a noise
> > range but it might be worth considering having different callbacks for
> > text and csv formats.
>
> Wow! Interesting. I tried the approach before but I didn't
> see any difference by the approach. But it may depend on my
> environment.
>
> I'll import the approach to the next patch set so that
> others can try the approach easily.
>
>
> Thanks,
> --
> kou
Hi Kou-san,
In the current implementation, there is no way that one can check
incompatibility
options in ProcessCopyOptions, we can postpone the check in CopyFromStart
or CopyToStart, but I think it is a little bit late. Do you think
adding an extra
check for incompatible options hook is acceptable (PFA)?
--
Regards
Junwang Zhao
Attachments:
[application/octet-stream] 0001-add-check-incomptiblity-options-hooks.patch (3.1K, ../../CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com/2-0001-add-check-incomptiblity-options-hooks.patch)
download | inline diff:
From f01ad61fe6c251a00870ace5d37669350f8e2043 Mon Sep 17 00:00:00 2001
From: Zhao Junwang <[email protected]>
Date: Fri, 26 Jan 2024 15:55:07 +0800
Subject: [PATCH] add check incomptiblity options hooks
Signed-off-by: Zhao Junwang <[email protected]>
---
src/backend/commands/copy.c | 5 +++++
src/include/commands/copyapi.h | 15 +++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 479f36868c..985d50870f 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -924,6 +924,11 @@ ProcessCopyOptions(ParseState *pstate,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("NULL specification and DEFAULT specification cannot be the same")));
}
+
+ if (is_from)
+ opts_out->from_routine->CopyFromCheckIncompatibleOptions(cstate, opts_out);
+ else
+ opts_out->to_routine->CopyToCheckIncompatibleOptions(cstate, opts_out);
}
/*
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 22accc83ab..34aa86761a 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -21,8 +21,10 @@
#include "nodes/parsenodes.h"
typedef struct CopyFromStateData *CopyFromState;
+typedef struct CopyFormatOptions;
typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel);
+typedef void (*CopyFromCheckIncompatibleOptions_function) (CopyFromState cstate, CopyFormatOptions *options);
typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate);
typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc);
typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
@@ -39,6 +41,12 @@ typedef struct CopyFromRoutine
*/
CopyFromProcessOption_function CopyFromProcessOption;
+ /*
+ * Called for processing incompatible options. This will report error
+ * messages when find incompatible options.
+ */
+ CopyFromCheckIncompatibleOptions_function CopyFromCheckIncompatibleOptions;
+
/*
* Called when COPY FROM is started. This will return a format as int16
* value. It's used for the CopyInResponse message.
@@ -67,6 +75,7 @@ extern CopyFromRoutine CopyFromRoutineBinary;
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
+typedef void (*CopyToCheckIncompatibleOptions_function) (CopyToState cstate, CopyFormatOptions *options);
typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
@@ -83,6 +92,12 @@ typedef struct CopyToRoutine
*/
CopyToProcessOption_function CopyToProcessOption;
+ /*
+ * Called for processing incompatible options. This will report error
+ * messages when find incompatible options.
+ */
+ CopyToCheckIncompatibleOptions_function CopyToCheckIncompatibleOptions;
+
/*
* Called when COPY TO is started. This will return a format as int16
* value. It's used for the CopyOutResponse message.
--
2.41.0
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-26 08:32 ` Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-26 08:32 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800,
Junwang Zhao <[email protected]> wrote:
> In the current implementation, there is no way that one can check
> incompatibility
> options in ProcessCopyOptions, we can postpone the check in CopyFromStart
> or CopyToStart, but I think it is a little bit late. Do you think
> adding an extra
> check for incompatible options hook is acceptable (PFA)?
Thanks for the suggestion! But I think that a custom handler
can do it in
CopyToProcessOption()/CopyFromProcessOption(). What do you
think about this? Or could you share a sample COPY TO/FROM
WITH() SQL you think?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-26 08:41 ` Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Junwang Zhao @ 2024-01-26 08:41 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jan 26, 2024 at 4:32 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3+-oG63GeG6v0L8EWi_8Fhuj9vJBhOteLxuBZwtun3GVA@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:18:14 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> > In the current implementation, there is no way that one can check
> > incompatibility
> > options in ProcessCopyOptions, we can postpone the check in CopyFromStart
> > or CopyToStart, but I think it is a little bit late. Do you think
> > adding an extra
> > check for incompatible options hook is acceptable (PFA)?
>
> Thanks for the suggestion! But I think that a custom handler
> can do it in
> CopyToProcessOption()/CopyFromProcessOption(). What do you
> think about this? Or could you share a sample COPY TO/FROM
> WITH() SQL you think?
CopyToProcessOption()/CopyFromProcessOption() can only handle
single option, and store the options in the opaque field, but it can not
check the relation of two options, for example, considering json format,
the `header` option can not be handled by these two functions.
I want to find a way when the user specifies the header option, customer
handler can error out.
>
>
> Thanks,
> --
> kou
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-26 08:55 ` Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-26 08:55 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
Junwang Zhao <[email protected]> wrote:
> CopyToProcessOption()/CopyFromProcessOption() can only handle
> single option, and store the options in the opaque field, but it can not
> check the relation of two options, for example, considering json format,
> the `header` option can not be handled by these two functions.
>
> I want to find a way when the user specifies the header option, customer
> handler can error out.
Ah, you want to use a built-in option (such as "header")
value from a custom handler, right? Hmm, it may be better
that we call CopyToProcessOption()/CopyFromProcessOption()
for all options including built-in options.
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-26 09:02 ` Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 2 replies; 51+ messages in thread
From: Junwang Zhao @ 2024-01-26 09:02 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > single option, and store the options in the opaque field, but it can not
> > check the relation of two options, for example, considering json format,
> > the `header` option can not be handled by these two functions.
> >
> > I want to find a way when the user specifies the header option, customer
> > handler can error out.
>
> Ah, you want to use a built-in option (such as "header")
> value from a custom handler, right? Hmm, it may be better
> that we call CopyToProcessOption()/CopyFromProcessOption()
> for all options including built-in options.
>
Hmm, still I don't think it can handle all cases, since we don't know
the sequence of the options, we need all the options been parsed
before we check the compatibility of the options, or customer
handlers will need complicated logic to resolve that, which might
lead to ugly code :(
>
> Thanks,
> --
> kou
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-27 06:15 ` Junwang Zhao <[email protected]>
2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
1 sibling, 1 reply; 51+ messages in thread
From: Junwang Zhao @ 2024-01-27 06:15 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi Kou-san,
On Fri, Jan 26, 2024 at 5:02 PM Junwang Zhao <[email protected]> wrote:
>
> On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > Junwang Zhao <[email protected]> wrote:
> >
> > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > single option, and store the options in the opaque field, but it can not
> > > check the relation of two options, for example, considering json format,
> > > the `header` option can not be handled by these two functions.
> > >
> > > I want to find a way when the user specifies the header option, customer
> > > handler can error out.
> >
> > Ah, you want to use a built-in option (such as "header")
> > value from a custom handler, right? Hmm, it may be better
> > that we call CopyToProcessOption()/CopyFromProcessOption()
> > for all options including built-in options.
> >
> Hmm, still I don't think it can handle all cases, since we don't know
> the sequence of the options, we need all the options been parsed
> before we check the compatibility of the options, or customer
> handlers will need complicated logic to resolve that, which might
> lead to ugly code :(
>
I have been working on a *COPY TO JSON* extension since yesterday,
which is based on your V6 patch set, I'd like to give you more input
so you can make better decisions about the implementation(with only
pg-copy-arrow you might not get everything considered).
V8 is based on V6, so anybody involved in the performance issue
should still review the V7 patch set.
0001-0008 is your original V6 implementations
0009 is some changes made by me, I changed CopyToGetFormat to
CopyToSendCopyBegin because pg_copy_json need to send different bytes
in SendCopyBegin, get the format code along is not enough, I once had
a thought that may be we should merge SendCopyBegin/SendCopyEnd into
CopyToStart/CopyToEnd but I don't do that in this patch. I have also
exported more APIs for extension usage.
00010 is the pg_copy_json extension, I think this should be a good
case which can utilize the *extendable copy format* feature, maybe we
should delete copy_test_format if we have this extension as an
example?
> >
> > Thanks,
> > --
> > kou
>
>
>
> --
> Regards
> Junwang Zhao
--
Regards
Junwang Zhao
Attachments:
[application/octet-stream] v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch (3.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/2-v8-0004-Add-support-for-implementing-custom-COPY-TO-forma.patch)
download | inline diff:
From 4b177469f3fb8f14f0cd6bff3c7878dcafd9b760 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 15:12:43 +0900
Subject: [PATCH v8 04/10] Add support for implementing custom COPY TO format
as extension
* Add CopyToStateData::opaque that can be used to keep data for custom
COPY TO format implementation
* Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf
* Rename CopySendEndOfRow() to CopyToStateFlush() because it's a
method for CopyToState and it's used for flushing. End-of-row related
codes were moved to CopyToTextSendEndOfRow().
---
src/backend/commands/copyto.c | 15 +++++++--------
src/include/commands/copyapi.h | 5 +++++
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index cfc74ee7b1..b5d8678394 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate);
static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
static void CopySendString(CopyToState cstate, const char *str);
static void CopySendChar(CopyToState cstate, char c);
-static void CopySendEndOfRow(CopyToState cstate);
static void CopySendInt32(CopyToState cstate, int32 val);
static void CopySendInt16(CopyToState cstate, int16 val);
@@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
default:
break;
}
- CopySendEndOfRow(cstate);
+ CopyToStateFlush(cstate);
}
static void
@@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
}
}
- CopySendEndOfRow(cstate);
+ CopyToStateFlush(cstate);
}
static void
@@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate)
/* Generate trailer for a binary copy */
CopySendInt16(cstate, -1);
/* Need to flush out the trailer */
- CopySendEndOfRow(cstate);
+ CopyToStateFlush(cstate);
}
CopyToRoutine CopyToRoutineText = {
@@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate)
* CopySendData sends output data to the destination (file or frontend)
* CopySendString does the same for null-terminated strings
* CopySendChar does the same for single characters
- * CopySendEndOfRow does the appropriate thing at end of each data row
- * (data is not actually flushed except by CopySendEndOfRow)
+ * CopyToStateFlush flushes the buffered data
+ * (data is not actually flushed except by CopyToStateFlush)
*
* NB: no data conversion is applied by these functions
*----------
@@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c)
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
}
-static void
-CopySendEndOfRow(CopyToState cstate)
+void
+CopyToStateFlush(CopyToState cstate)
{
StringInfo fe_msgbuf = cstate->fe_msgbuf;
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index a869d78d72..ffad433a21 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -174,6 +174,11 @@ typedef struct CopyToStateData
FmgrInfo *out_functions; /* lookup info for output functions */
MemoryContext rowcontext; /* per-row evaluation context */
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* For custom format implementation */
+ void *opaque; /* private space */
} CopyToStateData;
+extern void CopyToStateFlush(CopyToState cstate);
+
#endif /* COPYAPI_H */
--
2.41.0
[application/octet-stream] v8-0001-Extract-COPY-TO-format-implementations.patch (23.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/3-v8-0001-Extract-COPY-TO-format-implementations.patch)
download | inline diff:
From 6e68ba6380dc825a242e7f0d0a53442bba3a4a61 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Mon, 4 Dec 2023 12:32:54 +0900
Subject: [PATCH v8 01/10] Extract COPY TO format implementations
This is a part of making COPY format extendable. See also these past
discussions:
* New Copy Formats - avro/orc/parquet:
https://www.postgresql.org/message-id/flat/20180210151304.fonjztsynewldfba%40gmail.com
* Make COPY extendable in order to support Parquet and other formats:
https://www.postgresql.org/message-id/flat/CAJ7c6TM6Bz1c3F04Cy6%2BSzuWfKmr0kU8c_3Stnvh_8BR0D6k8Q%40mail.gmail.com
This doesn't change the current behavior. This just introduces
CopyToRoutine, which just has function pointers of format
implementation like TupleTableSlotOps, and use it for existing "text",
"csv" and "binary" format implementations.
Note that CopyToRoutine can't be used from extensions yet because
CopySend*() aren't exported yet. Extensions can't send formatted data
to a destination without CopySend*(). They will be exported by
subsequent patches.
Here is a benchmark result with/without this change because there was
a discussion that we should care about performance regression:
https://www.postgresql.org/message-id/3741749.1655952719%40sss.pgh.pa.us
> I think that step 1 ought to be to convert the existing formats into
> plug-ins, and demonstrate that there's no significant loss of
> performance.
You can see that there is no significant loss of performance:
Data: Random 32 bit integers:
CREATE TABLE data (int32 integer);
INSERT INTO data
SELECT random() * 10000
FROM generate_series(1, ${n_records});
The number of records: 100K, 1M and 10M
100K without this change:
format,elapsed time (ms)
text,11.002
csv,11.696
binary,11.352
100K with this change:
format,elapsed time (ms)
text,100000,11.562
csv,100000,11.889
binary,100000,10.825
1M without this change:
format,elapsed time (ms)
text,108.359
csv,114.233
binary,111.251
1M with this change:
format,elapsed time (ms)
text,111.269
csv,116.277
binary,104.765
10M without this change:
format,elapsed time (ms)
text,1090.763
csv,1136.103
binary,1137.141
10M with this change:
format,elapsed time (ms)
text,1082.654
csv,1196.991
binary,1069.697
---
contrib/file_fdw/file_fdw.c | 2 +-
src/backend/commands/copy.c | 43 +++-
src/backend/commands/copyfrom.c | 2 +-
src/backend/commands/copyto.c | 428 ++++++++++++++++++++------------
src/include/commands/copy.h | 7 +-
src/include/commands/copyapi.h | 59 +++++
6 files changed, 376 insertions(+), 165 deletions(-)
create mode 100644 src/include/commands/copyapi.h
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 249d82d3a0..9e4e819858 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -329,7 +329,7 @@ file_fdw_validator(PG_FUNCTION_ARGS)
/*
* Now apply the core COPY code's validation logic for more checks.
*/
- ProcessCopyOptions(NULL, NULL, true, other_options);
+ ProcessCopyOptions(NULL, NULL, true, NULL, other_options);
/*
* Either filename or program option is required for file_fdw foreign
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index cc0786c6f4..5f3697a5f9 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -442,6 +442,9 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from)
* a list of options. In that usage, 'opts_out' can be passed as NULL and
* the collected data is just leaked until CurrentMemoryContext is reset.
*
+ * 'cstate' is CopyToState* for !is_from, CopyFromState* for is_from. 'cstate'
+ * may be NULL. For example, file_fdw uses NULL.
+ *
* Note that additional checking, such as whether column names listed in FORCE
* QUOTE actually exist, has to be applied later. This just checks for
* self-consistency of the options list.
@@ -450,6 +453,7 @@ void
ProcessCopyOptions(ParseState *pstate,
CopyFormatOptions *opts_out,
bool is_from,
+ void *cstate,
List *options)
{
bool format_specified = false;
@@ -464,7 +468,13 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->file_encoding = -1;
- /* Extract options from the statement node tree */
+ /* Text is the default format. */
+ opts_out->to_routine = &CopyToRoutineText;
+
+ /*
+ * Extract only the "format" option to detect target routine as the first
+ * step
+ */
foreach(option, options)
{
DefElem *defel = lfirst_node(DefElem, option);
@@ -479,15 +489,29 @@ ProcessCopyOptions(ParseState *pstate,
if (strcmp(fmt, "text") == 0)
/* default format */ ;
else if (strcmp(fmt, "csv") == 0)
+ {
opts_out->csv_mode = true;
+ opts_out->to_routine = &CopyToRoutineCSV;
+ }
else if (strcmp(fmt, "binary") == 0)
+ {
opts_out->binary = true;
+ opts_out->to_routine = &CopyToRoutineBinary;
+ }
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("COPY format \"%s\" not recognized", fmt),
parser_errposition(pstate, defel->location)));
}
+ }
+ /* Extract options except "format" from the statement node tree */
+ foreach(option, options)
+ {
+ DefElem *defel = lfirst_node(DefElem, option);
+
+ if (strcmp(defel->defname, "format") == 0)
+ continue;
else if (strcmp(defel->defname, "freeze") == 0)
{
if (freeze_specified)
@@ -616,11 +640,18 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->on_error = defGetCopyOnErrorChoice(defel, pstate, is_from);
}
else
- ereport(ERROR,
- (errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("option \"%s\" not recognized",
- defel->defname),
- parser_errposition(pstate, defel->location)));
+ {
+ bool processed = false;
+
+ if (!is_from)
+ processed = opts_out->to_routine->CopyToProcessOption(cstate, defel);
+ if (!processed)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("option \"%s\" not recognized",
+ defel->defname),
+ parser_errposition(pstate, defel->location)));
+ }
}
/*
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index 1fe70b9133..fb3d4d9296 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1416,7 +1416,7 @@ BeginCopyFrom(ParseState *pstate,
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
/* Extract options from the statement node tree */
- ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , options);
+ ProcessCopyOptions(pstate, &cstate->opts, true /* is_from */ , cstate, options);
/* Process the target relation */
cstate->rel = rel;
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index d3dc3fc854..6547b7c654 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -131,6 +131,275 @@ static void CopySendEndOfRow(CopyToState cstate);
static void CopySendInt32(CopyToState cstate, int32 val);
static void CopySendInt16(CopyToState cstate, int16 val);
+/*
+ * CopyToRoutine implementations.
+ */
+
+/*
+ * CopyToRoutine implementation for "text" and "csv". CopyToText*()
+ * refer cstate->opts.csv_mode and change their behavior. We can split this
+ * implementation and stop referring cstate->opts.csv_mode later.
+ */
+
+/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
+ * move the code to here later. */
+static bool
+CopyToTextProcessOption(CopyToState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyToTextGetFormat(CopyToState cstate)
+{
+ return 0;
+}
+
+static void
+CopyToTextSendEndOfRow(CopyToState cstate)
+{
+ switch (cstate->copy_dest)
+ {
+ case COPY_FILE:
+ /* Default line termination depends on platform */
+#ifndef WIN32
+ CopySendChar(cstate, '\n');
+#else
+ CopySendString(cstate, "\r\n");
+#endif
+ break;
+ case COPY_FRONTEND:
+ /* The FE/BE protocol uses \n as newline for all platforms */
+ CopySendChar(cstate, '\n');
+ break;
+ default:
+ break;
+ }
+ CopySendEndOfRow(cstate);
+}
+
+static void
+CopyToTextStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ int num_phys_attrs;
+ ListCell *cur;
+
+ num_phys_attrs = tupDesc->natts;
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
+
+ /*
+ * For non-binary copy, we need to convert null_print to file encoding,
+ * because it will be sent directly with CopySendString.
+ */
+ if (cstate->need_transcoding)
+ cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
+ cstate->opts.null_print_len,
+ cstate->file_encoding);
+
+ /* if a header has been requested send the line */
+ if (cstate->opts.header_line)
+ {
+ bool hdr_delim = false;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ char *colname;
+
+ if (hdr_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ hdr_delim = true;
+
+ colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
+
+ if (cstate->opts.csv_mode)
+ CopyAttributeOutCSV(cstate, colname, false,
+ list_length(cstate->attnumlist) == 1);
+ else
+ CopyAttributeOutText(cstate, colname);
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+ }
+}
+
+static void
+CopyToTextOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ bool need_delim = false;
+ FmgrInfo *out_functions = cstate->out_functions;
+ ListCell *cur;
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Datum value = slot->tts_values[attnum - 1];
+ bool isnull = slot->tts_isnull[attnum - 1];
+
+ if (need_delim)
+ CopySendChar(cstate, cstate->opts.delim[0]);
+ need_delim = true;
+
+ if (isnull)
+ {
+ CopySendString(cstate, cstate->opts.null_print_client);
+ }
+ else
+ {
+ char *string;
+
+ string = OutputFunctionCall(&out_functions[attnum - 1], value);
+ if (cstate->opts.csv_mode)
+ CopyAttributeOutCSV(cstate, string,
+ cstate->opts.force_quote_flags[attnum - 1],
+ list_length(cstate->attnumlist) == 1);
+ else
+ CopyAttributeOutText(cstate, string);
+ }
+ }
+
+ CopyToTextSendEndOfRow(cstate);
+}
+
+static void
+CopyToTextEnd(CopyToState cstate)
+{
+}
+
+/*
+ * CopyToRoutine implementation for "binary".
+ */
+
+/* All "binary" options are parsed in ProcessCopyOptions(). We may move the
+ * code to here later. */
+static bool
+CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyToBinaryGetFormat(CopyToState cstate)
+{
+ return 1;
+}
+
+static void
+CopyToBinaryStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ int num_phys_attrs;
+ ListCell *cur;
+
+ num_phys_attrs = tupDesc->natts;
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
+
+ {
+ /* Generate header for a binary copy */
+ int32 tmp;
+
+ /* Signature */
+ CopySendData(cstate, BinarySignature, 11);
+ /* Flags field */
+ tmp = 0;
+ CopySendInt32(cstate, tmp);
+ /* No header extension */
+ tmp = 0;
+ CopySendInt32(cstate, tmp);
+ }
+}
+
+static void
+CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ FmgrInfo *out_functions = cstate->out_functions;
+ ListCell *cur;
+
+ /* Binary per-tuple header */
+ CopySendInt16(cstate, list_length(cstate->attnumlist));
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Datum value = slot->tts_values[attnum - 1];
+ bool isnull = slot->tts_isnull[attnum - 1];
+
+ if (isnull)
+ {
+ CopySendInt32(cstate, -1);
+ }
+ else
+ {
+ bytea *outputbytes;
+
+ outputbytes = SendFunctionCall(&out_functions[attnum - 1], value);
+ CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
+ CopySendData(cstate, VARDATA(outputbytes),
+ VARSIZE(outputbytes) - VARHDRSZ);
+ }
+ }
+
+ CopySendEndOfRow(cstate);
+}
+
+static void
+CopyToBinaryEnd(CopyToState cstate)
+{
+ /* Generate trailer for a binary copy */
+ CopySendInt16(cstate, -1);
+ /* Need to flush out the trailer */
+ CopySendEndOfRow(cstate);
+}
+
+CopyToRoutine CopyToRoutineText = {
+ .CopyToProcessOption = CopyToTextProcessOption,
+ .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToStart = CopyToTextStart,
+ .CopyToOneRow = CopyToTextOneRow,
+ .CopyToEnd = CopyToTextEnd,
+};
+
+/*
+ * We can use the same CopyToRoutine for both of "text" and "csv" because
+ * CopyToText*() refer cstate->opts.csv_mode and change their behavior. We can
+ * split the implementations and stop referring cstate->opts.csv_mode later.
+ */
+CopyToRoutine CopyToRoutineCSV = {
+ .CopyToProcessOption = CopyToTextProcessOption,
+ .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToStart = CopyToTextStart,
+ .CopyToOneRow = CopyToTextOneRow,
+ .CopyToEnd = CopyToTextEnd,
+};
+
+CopyToRoutine CopyToRoutineBinary = {
+ .CopyToProcessOption = CopyToBinaryProcessOption,
+ .CopyToGetFormat = CopyToBinaryGetFormat,
+ .CopyToStart = CopyToBinaryStart,
+ .CopyToOneRow = CopyToBinaryOneRow,
+ .CopyToEnd = CopyToBinaryEnd,
+};
/*
* Send copy start/stop messages for frontend copies. These have changed
@@ -141,7 +410,7 @@ SendCopyBegin(CopyToState cstate)
{
StringInfoData buf;
int natts = list_length(cstate->attnumlist);
- int16 format = (cstate->opts.binary ? 1 : 0);
+ int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate);
int i;
pq_beginmessage(&buf, PqMsg_CopyOutResponse);
@@ -198,16 +467,6 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
case COPY_FILE:
- if (!cstate->opts.binary)
- {
- /* Default line termination depends on platform */
-#ifndef WIN32
- CopySendChar(cstate, '\n');
-#else
- CopySendString(cstate, "\r\n");
-#endif
- }
-
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
ferror(cstate->copy_file))
@@ -242,10 +501,6 @@ CopySendEndOfRow(CopyToState cstate)
}
break;
case COPY_FRONTEND:
- /* The FE/BE protocol uses \n as newline for all platforms */
- if (!cstate->opts.binary)
- CopySendChar(cstate, '\n');
-
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
@@ -431,7 +686,7 @@ BeginCopyTo(ParseState *pstate,
oldcontext = MemoryContextSwitchTo(cstate->copycontext);
/* Extract options from the statement node tree */
- ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , options);
+ ProcessCopyOptions(pstate, &cstate->opts, false /* is_from */ , cstate, options);
/* Process the source/target relation or query */
if (rel)
@@ -748,8 +1003,6 @@ DoCopyTo(CopyToState cstate)
bool pipe = (cstate->filename == NULL && cstate->data_dest_cb == NULL);
bool fe_copy = (pipe && whereToSendOutput == DestRemote);
TupleDesc tupDesc;
- int num_phys_attrs;
- ListCell *cur;
uint64 processed;
if (fe_copy)
@@ -759,32 +1012,11 @@ DoCopyTo(CopyToState cstate)
tupDesc = RelationGetDescr(cstate->rel);
else
tupDesc = cstate->queryDesc->tupDesc;
- num_phys_attrs = tupDesc->natts;
cstate->opts.null_print_client = cstate->opts.null_print; /* default */
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
cstate->fe_msgbuf = makeStringInfo();
- /* Get info about the columns we need to process. */
- cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- Oid out_func_oid;
- bool isvarlena;
- Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
-
- if (cstate->opts.binary)
- getTypeBinaryOutputInfo(attr->atttypid,
- &out_func_oid,
- &isvarlena);
- else
- getTypeOutputInfo(attr->atttypid,
- &out_func_oid,
- &isvarlena);
- fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
- }
-
/*
* Create a temporary memory context that we can reset once per row to
* recover palloc'd memory. This avoids any problems with leaks inside
@@ -795,57 +1027,7 @@ DoCopyTo(CopyToState cstate)
"COPY TO",
ALLOCSET_DEFAULT_SIZES);
- if (cstate->opts.binary)
- {
- /* Generate header for a binary copy */
- int32 tmp;
-
- /* Signature */
- CopySendData(cstate, BinarySignature, 11);
- /* Flags field */
- tmp = 0;
- CopySendInt32(cstate, tmp);
- /* No header extension */
- tmp = 0;
- CopySendInt32(cstate, tmp);
- }
- else
- {
- /*
- * For non-binary copy, we need to convert null_print to file
- * encoding, because it will be sent directly with CopySendString.
- */
- if (cstate->need_transcoding)
- cstate->opts.null_print_client = pg_server_to_any(cstate->opts.null_print,
- cstate->opts.null_print_len,
- cstate->file_encoding);
-
- /* if a header has been requested send the line */
- if (cstate->opts.header_line)
- {
- bool hdr_delim = false;
-
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- char *colname;
-
- if (hdr_delim)
- CopySendChar(cstate, cstate->opts.delim[0]);
- hdr_delim = true;
-
- colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname);
-
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, colname, false,
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, colname);
- }
-
- CopySendEndOfRow(cstate);
- }
- }
+ cstate->opts.to_routine->CopyToStart(cstate, tupDesc);
if (cstate->rel)
{
@@ -884,13 +1066,7 @@ DoCopyTo(CopyToState cstate)
processed = ((DR_copy *) cstate->queryDesc->dest)->processed;
}
- if (cstate->opts.binary)
- {
- /* Generate trailer for a binary copy */
- CopySendInt16(cstate, -1);
- /* Need to flush out the trailer */
- CopySendEndOfRow(cstate);
- }
+ cstate->opts.to_routine->CopyToEnd(cstate);
MemoryContextDelete(cstate->rowcontext);
@@ -906,71 +1082,15 @@ DoCopyTo(CopyToState cstate)
static void
CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot)
{
- bool need_delim = false;
- FmgrInfo *out_functions = cstate->out_functions;
MemoryContext oldcontext;
- ListCell *cur;
- char *string;
MemoryContextReset(cstate->rowcontext);
oldcontext = MemoryContextSwitchTo(cstate->rowcontext);
- if (cstate->opts.binary)
- {
- /* Binary per-tuple header */
- CopySendInt16(cstate, list_length(cstate->attnumlist));
- }
-
/* Make sure the tuple is fully deconstructed */
slot_getallattrs(slot);
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- Datum value = slot->tts_values[attnum - 1];
- bool isnull = slot->tts_isnull[attnum - 1];
-
- if (!cstate->opts.binary)
- {
- if (need_delim)
- CopySendChar(cstate, cstate->opts.delim[0]);
- need_delim = true;
- }
-
- if (isnull)
- {
- if (!cstate->opts.binary)
- CopySendString(cstate, cstate->opts.null_print_client);
- else
- CopySendInt32(cstate, -1);
- }
- else
- {
- if (!cstate->opts.binary)
- {
- string = OutputFunctionCall(&out_functions[attnum - 1],
- value);
- if (cstate->opts.csv_mode)
- CopyAttributeOutCSV(cstate, string,
- cstate->opts.force_quote_flags[attnum - 1],
- list_length(cstate->attnumlist) == 1);
- else
- CopyAttributeOutText(cstate, string);
- }
- else
- {
- bytea *outputbytes;
-
- outputbytes = SendFunctionCall(&out_functions[attnum - 1],
- value);
- CopySendInt32(cstate, VARSIZE(outputbytes) - VARHDRSZ);
- CopySendData(cstate, VARDATA(outputbytes),
- VARSIZE(outputbytes) - VARHDRSZ);
- }
- }
- }
-
- CopySendEndOfRow(cstate);
+ cstate->opts.to_routine->CopyToOneRow(cstate, slot);
MemoryContextSwitchTo(oldcontext);
}
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index b3da3cb0be..34bea880ca 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -14,6 +14,7 @@
#ifndef COPY_H
#define COPY_H
+#include "commands/copyapi.h"
#include "nodes/execnodes.h"
#include "nodes/parsenodes.h"
#include "parser/parse_node.h"
@@ -74,11 +75,11 @@ typedef struct CopyFormatOptions
bool convert_selectively; /* do selective binary conversion? */
CopyOnErrorChoice on_error; /* what to do when error happened */
List *convert_select; /* list of column names (can be NIL) */
+ CopyToRoutine *to_routine; /* callback routines for COPY TO */
} CopyFormatOptions;
-/* These are private in commands/copy[from|to].c */
+/* This is private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
-typedef struct CopyToStateData *CopyToState;
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
typedef void (*copy_data_dest_cb) (void *data, int len);
@@ -87,7 +88,7 @@ extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
uint64 *processed);
-extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, List *options);
+extern void ProcessCopyOptions(ParseState *pstate, CopyFormatOptions *opts_out, bool is_from, void *cstate, List *options);
extern CopyFromState BeginCopyFrom(ParseState *pstate, Relation rel, Node *whereClause,
const char *filename,
bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options);
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
new file mode 100644
index 0000000000..eb68f2fb7b
--- /dev/null
+++ b/src/include/commands/copyapi.h
@@ -0,0 +1,59 @@
+/*-------------------------------------------------------------------------
+ *
+ * copyapi.h
+ * API for COPY TO/FROM handlers
+ *
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/commands/copyapi.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef COPYAPI_H
+#define COPYAPI_H
+
+#include "executor/tuptable.h"
+#include "nodes/parsenodes.h"
+
+/* This is private in commands/copyto.c */
+typedef struct CopyToStateData *CopyToState;
+
+typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
+typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
+typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
+typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
+typedef void (*CopyToEnd_function) (CopyToState cstate);
+
+/* Routines for a COPY TO format implementation. */
+typedef struct CopyToRoutine
+{
+ /*
+ * Called for processing one COPY TO option. This will return false when
+ * the given option is invalid.
+ */
+ CopyToProcessOption_function CopyToProcessOption;
+
+ /*
+ * Called when COPY TO is started. This will return a format as int16
+ * value. It's used for the CopyOutResponse message.
+ */
+ CopyToGetFormat_function CopyToGetFormat;
+
+ /* Called when COPY TO is started. This will send a header. */
+ CopyToStart_function CopyToStart;
+
+ /* Copy one row for COPY TO. */
+ CopyToOneRow_function CopyToOneRow;
+
+ /* Called when COPY TO is ended. This will send a trailer. */
+ CopyToEnd_function CopyToEnd;
+} CopyToRoutine;
+
+/* Built-in CopyToRoutine for "text", "csv" and "binary". */
+extern CopyToRoutine CopyToRoutineText;
+extern CopyToRoutine CopyToRoutineCSV;
+extern CopyToRoutine CopyToRoutineBinary;
+
+#endif /* COPYAPI_H */
--
2.41.0
[application/octet-stream] v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch (17.7K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/4-v8-0002-Add-support-for-adding-custom-COPY-TO-format.patch)
download | inline diff:
From a597f8a2beec12971d77419f08b5722f531774f3 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 13:58:38 +0900
Subject: [PATCH v8 02/10] Add support for adding custom COPY TO format
This uses the handler approach like tablesample. The approach creates
an internal function that returns an internal struct. In this case,
a COPY TO handler returns a CopyToRoutine.
We will add support for custom COPY FROM format later. We'll use the
same handler for COPY TO and COPY FROM. PostgreSQL calls a COPY
TO/FROM handler with "is_from" argument. It's true for COPY FROM and
false for COPY TO:
copy_handler(true) returns CopyToRoutine
copy_handler(false) returns CopyFromRoutine (not exist yet)
We discussed that we introduce a wrapper struct for it:
typedef struct CopyRoutine
{
NodeTag type;
/* either CopyToRoutine or CopyFromRoutine */
Node *routine;
}
copy_handler(true) returns CopyRoutine with CopyToRoutine
copy_handler(false) returns CopyRoutine with CopyFromRoutine
See also: https://www.postgresql.org/message-id/flat/CAD21AoCunywHird3GaPzWe6s9JG1wzxj3Cr6vGN36DDheGjOjA%40mail.gmail.com
But I noticed that we don't need the wrapper struct. We can just
CopyToRoutine or CopyFromRoutine. Because we can distinct the returned
struct by checking its NodeTag. So I don't use the wrapper struct
approach.
---
src/backend/commands/copy.c | 84 ++++++++++++++-----
src/backend/nodes/Makefile | 1 +
src/backend/nodes/gen_node_support.pl | 2 +
src/backend/utils/adt/pseudotypes.c | 1 +
src/include/catalog/pg_proc.dat | 6 ++
src/include/catalog/pg_type.dat | 6 ++
src/include/commands/copyapi.h | 2 +
src/include/nodes/meson.build | 1 +
src/test/modules/Makefile | 1 +
src/test/modules/meson.build | 1 +
src/test/modules/test_copy_format/.gitignore | 4 +
src/test/modules/test_copy_format/Makefile | 23 +++++
.../expected/test_copy_format.out | 17 ++++
src/test/modules/test_copy_format/meson.build | 33 ++++++++
.../test_copy_format/sql/test_copy_format.sql | 8 ++
.../test_copy_format--1.0.sql | 8 ++
.../test_copy_format/test_copy_format.c | 77 +++++++++++++++++
.../test_copy_format/test_copy_format.control | 4 +
18 files changed, 260 insertions(+), 19 deletions(-)
mode change 100644 => 100755 src/backend/nodes/gen_node_support.pl
create mode 100644 src/test/modules/test_copy_format/.gitignore
create mode 100644 src/test/modules/test_copy_format/Makefile
create mode 100644 src/test/modules/test_copy_format/expected/test_copy_format.out
create mode 100644 src/test/modules/test_copy_format/meson.build
create mode 100644 src/test/modules/test_copy_format/sql/test_copy_format.sql
create mode 100644 src/test/modules/test_copy_format/test_copy_format--1.0.sql
create mode 100644 src/test/modules/test_copy_format/test_copy_format.c
create mode 100644 src/test/modules/test_copy_format/test_copy_format.control
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 5f3697a5f9..6f0db0ae7c 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -32,6 +32,7 @@
#include "parser/parse_coerce.h"
#include "parser/parse_collate.h"
#include "parser/parse_expr.h"
+#include "parser/parse_func.h"
#include "parser/parse_relation.h"
#include "rewrite/rewriteHandler.h"
#include "utils/acl.h"
@@ -430,6 +431,69 @@ defGetCopyOnErrorChoice(DefElem *def, ParseState *pstate, bool is_from)
return COPY_ON_ERROR_STOP; /* keep compiler quiet */
}
+/*
+ * Process the "format" option.
+ *
+ * This function checks whether the option value is a built-in format such as
+ * "text" and "csv" or not. If the option value isn't a built-in format, this
+ * function finds a COPY format handler that returns a CopyToRoutine. If no
+ * COPY format handler is found, this function reports an error.
+ */
+static void
+ProcessCopyOptionCustomFormat(ParseState *pstate,
+ CopyFormatOptions *opts_out,
+ bool is_from,
+ DefElem *defel)
+{
+ char *format;
+ Oid funcargtypes[1];
+ Oid handlerOid = InvalidOid;
+ Datum datum;
+ void *routine;
+
+ format = defGetString(defel);
+
+ /* built-in formats */
+ if (strcmp(format, "text") == 0)
+ /* default format */ return;
+ else if (strcmp(format, "csv") == 0)
+ {
+ opts_out->csv_mode = true;
+ opts_out->to_routine = &CopyToRoutineCSV;
+ return;
+ }
+ else if (strcmp(format, "binary") == 0)
+ {
+ opts_out->binary = true;
+ opts_out->to_routine = &CopyToRoutineBinary;
+ return;
+ }
+
+ /* custom format */
+ if (!is_from)
+ {
+ funcargtypes[0] = INTERNALOID;
+ handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
+ funcargtypes, true);
+ }
+ if (!OidIsValid(handlerOid))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY format \"%s\" not recognized", format),
+ parser_errposition(pstate, defel->location)));
+
+ datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from));
+ routine = DatumGetPointer(datum);
+ if (routine == NULL || !IsA(routine, CopyToRoutine))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct",
+ format, handlerOid),
+ parser_errposition(pstate, defel->location)));
+
+ opts_out->to_routine = routine;
+}
+
/*
* Process the statement option list for COPY.
*
@@ -481,28 +545,10 @@ ProcessCopyOptions(ParseState *pstate,
if (strcmp(defel->defname, "format") == 0)
{
- char *fmt = defGetString(defel);
-
if (format_specified)
errorConflictingDefElem(defel, pstate);
format_specified = true;
- if (strcmp(fmt, "text") == 0)
- /* default format */ ;
- else if (strcmp(fmt, "csv") == 0)
- {
- opts_out->csv_mode = true;
- opts_out->to_routine = &CopyToRoutineCSV;
- }
- else if (strcmp(fmt, "binary") == 0)
- {
- opts_out->binary = true;
- opts_out->to_routine = &CopyToRoutineBinary;
- }
- else
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format \"%s\" not recognized", fmt),
- parser_errposition(pstate, defel->location)));
+ ProcessCopyOptionCustomFormat(pstate, opts_out, is_from, defel);
}
}
/* Extract options except "format" from the statement node tree */
diff --git a/src/backend/nodes/Makefile b/src/backend/nodes/Makefile
index 66bbad8e6e..173ee11811 100644
--- a/src/backend/nodes/Makefile
+++ b/src/backend/nodes/Makefile
@@ -49,6 +49,7 @@ node_headers = \
access/sdir.h \
access/tableam.h \
access/tsmapi.h \
+ commands/copyapi.h \
commands/event_trigger.h \
commands/trigger.h \
executor/tuptable.h \
diff --git a/src/backend/nodes/gen_node_support.pl b/src/backend/nodes/gen_node_support.pl
old mode 100644
new mode 100755
index 2f0a59bc87..bd397f45ac
--- a/src/backend/nodes/gen_node_support.pl
+++ b/src/backend/nodes/gen_node_support.pl
@@ -61,6 +61,7 @@ my @all_input_files = qw(
access/sdir.h
access/tableam.h
access/tsmapi.h
+ commands/copyapi.h
commands/event_trigger.h
commands/trigger.h
executor/tuptable.h
@@ -85,6 +86,7 @@ my @nodetag_only_files = qw(
access/sdir.h
access/tableam.h
access/tsmapi.h
+ commands/copyapi.h
commands/event_trigger.h
commands/trigger.h
executor/tuptable.h
diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c
index a3a991f634..d308780c43 100644
--- a/src/backend/utils/adt/pseudotypes.c
+++ b/src/backend/utils/adt/pseudotypes.c
@@ -373,6 +373,7 @@ PSEUDOTYPE_DUMMY_IO_FUNCS(fdw_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(table_am_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(index_am_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(tsm_handler);
+PSEUDOTYPE_DUMMY_IO_FUNCS(copy_handler);
PSEUDOTYPE_DUMMY_IO_FUNCS(internal);
PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement);
PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 29af4ce65d..d4e426687c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7617,6 +7617,12 @@
{ oid => '3312', descr => 'I/O',
proname => 'tsm_handler_out', prorettype => 'cstring',
proargtypes => 'tsm_handler', prosrc => 'tsm_handler_out' },
+{ oid => '8753', descr => 'I/O',
+ proname => 'copy_handler_in', proisstrict => 'f', prorettype => 'copy_handler',
+ proargtypes => 'cstring', prosrc => 'copy_handler_in' },
+{ oid => '8754', descr => 'I/O',
+ proname => 'copy_handler_out', prorettype => 'cstring',
+ proargtypes => 'copy_handler', prosrc => 'copy_handler_out' },
{ oid => '267', descr => 'I/O',
proname => 'table_am_handler_in', proisstrict => 'f',
prorettype => 'table_am_handler', proargtypes => 'cstring',
diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat
index d29194da31..2040d5da83 100644
--- a/src/include/catalog/pg_type.dat
+++ b/src/include/catalog/pg_type.dat
@@ -632,6 +632,12 @@
typcategory => 'P', typinput => 'tsm_handler_in',
typoutput => 'tsm_handler_out', typreceive => '-', typsend => '-',
typalign => 'i' },
+{ oid => '8752',
+ descr => 'pseudo-type for the result of a copy to/from method functoin',
+ typname => 'copy_handler', typlen => '4', typbyval => 't', typtype => 'p',
+ typcategory => 'P', typinput => 'copy_handler_in',
+ typoutput => 'copy_handler_out', typreceive => '-', typsend => '-',
+ typalign => 'i' },
{ oid => '269',
typname => 'table_am_handler',
descr => 'pseudo-type for the result of a table AM handler function',
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index eb68f2fb7b..9c25e1c415 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -29,6 +29,8 @@ typedef void (*CopyToEnd_function) (CopyToState cstate);
/* Routines for a COPY TO format implementation. */
typedef struct CopyToRoutine
{
+ NodeTag type;
+
/*
* Called for processing one COPY TO option. This will return false when
* the given option is invalid.
diff --git a/src/include/nodes/meson.build b/src/include/nodes/meson.build
index b665e55b65..103df1a787 100644
--- a/src/include/nodes/meson.build
+++ b/src/include/nodes/meson.build
@@ -11,6 +11,7 @@ node_support_input_i = [
'access/sdir.h',
'access/tableam.h',
'access/tsmapi.h',
+ 'commands/copyapi.h',
'commands/event_trigger.h',
'commands/trigger.h',
'executor/tuptable.h',
diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index e32c8925f6..9d57b868d5 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -15,6 +15,7 @@ SUBDIRS = \
spgist_name_ops \
test_bloomfilter \
test_copy_callbacks \
+ test_copy_format \
test_custom_rmgrs \
test_ddl_deparse \
test_dsa \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 397e0906e6..d76f2a6003 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -13,6 +13,7 @@ subdir('spgist_name_ops')
subdir('ssl_passphrase_callback')
subdir('test_bloomfilter')
subdir('test_copy_callbacks')
+subdir('test_copy_format')
subdir('test_custom_rmgrs')
subdir('test_ddl_deparse')
subdir('test_dsa')
diff --git a/src/test/modules/test_copy_format/.gitignore b/src/test/modules/test_copy_format/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_copy_format/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_copy_format/Makefile b/src/test/modules/test_copy_format/Makefile
new file mode 100644
index 0000000000..8497f91624
--- /dev/null
+++ b/src/test/modules/test_copy_format/Makefile
@@ -0,0 +1,23 @@
+# src/test/modules/test_copy_format/Makefile
+
+MODULE_big = test_copy_format
+OBJS = \
+ $(WIN32RES) \
+ test_copy_format.o
+PGFILEDESC = "test_copy_format - test custom COPY FORMAT"
+
+EXTENSION = test_copy_format
+DATA = test_copy_format--1.0.sql
+
+REGRESS = test_copy_format
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_copy_format
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out
new file mode 100644
index 0000000000..3a24ae7b97
--- /dev/null
+++ b/src/test/modules/test_copy_format/expected/test_copy_format.out
@@ -0,0 +1,17 @@
+CREATE EXTENSION test_copy_format;
+CREATE TABLE public.test (a INT, b INT, c INT);
+INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test TO stdout WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
+NOTICE: test_copy_format: is_from=false
+NOTICE: CopyToProcessOption: "option_before"="before"
+NOTICE: CopyToProcessOption: "option_after"="after"
+NOTICE: CopyToGetFormat
+NOTICE: CopyToStart: natts=3
+NOTICE: CopyToOneRow: tts_nvalid=3
+NOTICE: CopyToOneRow: tts_nvalid=3
+NOTICE: CopyToOneRow: tts_nvalid=3
+NOTICE: CopyToEnd
diff --git a/src/test/modules/test_copy_format/meson.build b/src/test/modules/test_copy_format/meson.build
new file mode 100644
index 0000000000..4cefe7b709
--- /dev/null
+++ b/src/test/modules/test_copy_format/meson.build
@@ -0,0 +1,33 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+test_copy_format_sources = files(
+ 'test_copy_format.c',
+)
+
+if host_system == 'windows'
+ test_copy_format_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'test_copy_format',
+ '--FILEDESC', 'test_copy_format - test custom COPY FORMAT',])
+endif
+
+test_copy_format = shared_module('test_copy_format',
+ test_copy_format_sources,
+ kwargs: pg_test_mod_args,
+)
+test_install_libs += test_copy_format
+
+test_install_data += files(
+ 'test_copy_format.control',
+ 'test_copy_format--1.0.sql',
+)
+
+tests += {
+ 'name': 'test_copy_format',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'test_copy_format',
+ ],
+ },
+}
diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql
new file mode 100644
index 0000000000..0eb7ed2e11
--- /dev/null
+++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql
@@ -0,0 +1,8 @@
+CREATE EXTENSION test_copy_format;
+CREATE TABLE public.test (a INT, b INT, c INT);
+INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test TO stdout WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
diff --git a/src/test/modules/test_copy_format/test_copy_format--1.0.sql b/src/test/modules/test_copy_format/test_copy_format--1.0.sql
new file mode 100644
index 0000000000..d24ea03ce9
--- /dev/null
+++ b/src/test/modules/test_copy_format/test_copy_format--1.0.sql
@@ -0,0 +1,8 @@
+/* src/test/modules/test_copy_format/test_copy_format--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_copy_format" to load this file. \quit
+
+CREATE FUNCTION test_copy_format(internal)
+ RETURNS copy_handler
+ AS 'MODULE_PATHNAME' LANGUAGE C;
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
new file mode 100644
index 0000000000..a2219afcde
--- /dev/null
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -0,0 +1,77 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_copy_format.c
+ * Code for testing custom COPY format.
+ *
+ * Portions Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_copy_format/test_copy_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/defrem.h"
+
+PG_MODULE_MAGIC;
+
+static bool
+CopyToProcessOption(CopyToState cstate, DefElem *defel)
+{
+ ereport(NOTICE,
+ (errmsg("CopyToProcessOption: \"%s\"=\"%s\"",
+ defel->defname, defGetString(defel))));
+ return true;
+}
+
+static int16
+CopyToGetFormat(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToGetFormat")));
+ return 0;
+}
+
+static void
+CopyToStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ ereport(NOTICE, (errmsg("CopyToStart: natts=%d", tupDesc->natts)));
+}
+
+static void
+CopyToOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ ereport(NOTICE, (errmsg("CopyToOneRow: tts_nvalid=%u", slot->tts_nvalid)));
+}
+
+static void
+CopyToEnd(CopyToState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyToEnd")));
+}
+
+static const CopyToRoutine CopyToRoutineTestCopyFormat = {
+ .type = T_CopyToRoutine,
+ .CopyToProcessOption = CopyToProcessOption,
+ .CopyToGetFormat = CopyToGetFormat,
+ .CopyToStart = CopyToStart,
+ .CopyToOneRow = CopyToOneRow,
+ .CopyToEnd = CopyToEnd,
+};
+
+PG_FUNCTION_INFO_V1(test_copy_format);
+Datum
+test_copy_format(PG_FUNCTION_ARGS)
+{
+ bool is_from = PG_GETARG_BOOL(0);
+
+ ereport(NOTICE,
+ (errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false")));
+
+ if (is_from)
+ elog(ERROR, "COPY FROM isn't supported yet");
+
+ PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat);
+}
diff --git a/src/test/modules/test_copy_format/test_copy_format.control b/src/test/modules/test_copy_format/test_copy_format.control
new file mode 100644
index 0000000000..f05a636235
--- /dev/null
+++ b/src/test/modules/test_copy_format/test_copy_format.control
@@ -0,0 +1,4 @@
+comment = 'Test code for custom COPY format'
+default_version = '1.0'
+module_pathname = '$libdir/test_copy_format'
+relocatable = true
--
2.41.0
[application/octet-stream] v8-0003-Export-CopyToStateData.patch (13.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/5-v8-0003-Export-CopyToStateData.patch)
download | inline diff:
From c3a59753b1157dc8e47e719263f58677acc33178 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 14:54:10 +0900
Subject: [PATCH v8 03/10] Export CopyToStateData
It's for custom COPY TO format handlers implemented as extension.
This just moves codes. This doesn't change codes except CopyDest enum
values. CopyDest enum values such as COPY_FILE are conflicted
CopySource enum values defined in copyfrom_internal.h. So COPY_DEST_
prefix instead of COPY_ prefix is used. For example, COPY_FILE is
renamed to COPY_DEST_FILE.
Note that this change isn't enough to implement a custom COPY TO
format handler as extension. We'll do the followings in a subsequent
commit:
1. Add an opaque space for custom COPY TO format handler
2. Export CopySendEndOfRow() to flush buffer
---
src/backend/commands/copyto.c | 74 +++-----------------
src/include/commands/copy.h | 59 ----------------
src/include/commands/copyapi.h | 120 ++++++++++++++++++++++++++++++++-
3 files changed, 127 insertions(+), 126 deletions(-)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index 6547b7c654..cfc74ee7b1 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -43,64 +43,6 @@
#include "utils/rel.h"
#include "utils/snapmgr.h"
-/*
- * Represents the different dest cases we need to worry about at
- * the bottom level
- */
-typedef enum CopyDest
-{
- COPY_FILE, /* to file (or a piped program) */
- COPY_FRONTEND, /* to frontend */
- COPY_CALLBACK, /* to callback function */
-} CopyDest;
-
-/*
- * This struct contains all the state variables used throughout a COPY TO
- * operation.
- *
- * Multi-byte encodings: all supported client-side encodings encode multi-byte
- * characters by having the first byte's high bit set. Subsequent bytes of the
- * character can have the high bit not set. When scanning data in such an
- * encoding to look for a match to a single-byte (ie ASCII) character, we must
- * use the full pg_encoding_mblen() machinery to skip over multibyte
- * characters, else we might find a false match to a trailing byte. In
- * supported server encodings, there is no possibility of a false match, and
- * it's faster to make useless comparisons to trailing bytes than it is to
- * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
- * when we have to do it the hard way.
- */
-typedef struct CopyToStateData
-{
- /* low-level state data */
- CopyDest copy_dest; /* type of copy source/destination */
- FILE *copy_file; /* used if copy_dest == COPY_FILE */
- StringInfo fe_msgbuf; /* used for all dests during COPY TO */
-
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy to */
- QueryDesc *queryDesc; /* executable query to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDOUT */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_dest_cb data_dest_cb; /* function for writing data */
-
- CopyFormatOptions opts;
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- FmgrInfo *out_functions; /* lookup info for output functions */
- MemoryContext rowcontext; /* per-row evaluation context */
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyToStateData;
-
/* DestReceiver for COPY (query) TO */
typedef struct
{
@@ -160,7 +102,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
{
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
/* Default line termination depends on platform */
#ifndef WIN32
CopySendChar(cstate, '\n');
@@ -168,7 +110,7 @@ CopyToTextSendEndOfRow(CopyToState cstate)
CopySendString(cstate, "\r\n");
#endif
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* The FE/BE protocol uses \n as newline for all platforms */
CopySendChar(cstate, '\n');
break;
@@ -419,7 +361,7 @@ SendCopyBegin(CopyToState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_dest = COPY_FRONTEND;
+ cstate->copy_dest = COPY_DEST_FRONTEND;
}
static void
@@ -466,7 +408,7 @@ CopySendEndOfRow(CopyToState cstate)
switch (cstate->copy_dest)
{
- case COPY_FILE:
+ case COPY_DEST_FILE:
if (fwrite(fe_msgbuf->data, fe_msgbuf->len, 1,
cstate->copy_file) != 1 ||
ferror(cstate->copy_file))
@@ -500,11 +442,11 @@ CopySendEndOfRow(CopyToState cstate)
errmsg("could not write to COPY file: %m")));
}
break;
- case COPY_FRONTEND:
+ case COPY_DEST_FRONTEND:
/* Dump the accumulated row as one CopyData message */
(void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len);
break;
- case COPY_CALLBACK:
+ case COPY_DEST_CALLBACK:
cstate->data_dest_cb(fe_msgbuf->data, fe_msgbuf->len);
break;
}
@@ -877,12 +819,12 @@ BeginCopyTo(ParseState *pstate,
/* See Multibyte encoding comment above */
cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding);
- cstate->copy_dest = COPY_FILE; /* default */
+ cstate->copy_dest = COPY_DEST_FILE; /* default */
if (data_dest_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_dest = COPY_CALLBACK;
+ cstate->copy_dest = COPY_DEST_CALLBACK;
cstate->data_dest_cb = data_dest_cb;
}
else if (pipe)
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 34bea880ca..b3f4682f95 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -20,69 +20,10 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
-/*
- * Represents whether a header line should be present, and whether it must
- * match the actual names (which implies "true").
- */
-typedef enum CopyHeaderChoice
-{
- COPY_HEADER_FALSE = 0,
- COPY_HEADER_TRUE,
- COPY_HEADER_MATCH,
-} CopyHeaderChoice;
-
-/*
- * Represents where to save input processing errors. More values to be added
- * in the future.
- */
-typedef enum CopyOnErrorChoice
-{
- COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */
- COPY_ON_ERROR_IGNORE, /* ignore errors */
-} CopyOnErrorChoice;
-
-/*
- * A struct to hold COPY options, in a parsed form. All of these are related
- * to formatting, except for 'freeze', which doesn't really belong here, but
- * it's expedient to parse it along with all the other options.
- */
-typedef struct CopyFormatOptions
-{
- /* parameters from the COPY command */
- int file_encoding; /* file or remote side's character encoding,
- * -1 if not specified */
- bool binary; /* binary format? */
- bool freeze; /* freeze rows on loading? */
- bool csv_mode; /* Comma Separated Value format? */
- CopyHeaderChoice header_line; /* header line? */
- char *null_print; /* NULL marker string (server encoding!) */
- int null_print_len; /* length of same */
- char *null_print_client; /* same converted to file encoding */
- char *default_print; /* DEFAULT marker string */
- int default_print_len; /* length of same */
- char *delim; /* column delimiter (must be 1 byte) */
- char *quote; /* CSV quote char (must be 1 byte) */
- char *escape; /* CSV escape char (must be 1 byte) */
- List *force_quote; /* list of column names */
- bool force_quote_all; /* FORCE_QUOTE *? */
- bool *force_quote_flags; /* per-column CSV FQ flags */
- List *force_notnull; /* list of column names */
- bool force_notnull_all; /* FORCE_NOT_NULL *? */
- bool *force_notnull_flags; /* per-column CSV FNN flags */
- List *force_null; /* list of column names */
- bool force_null_all; /* FORCE_NULL *? */
- bool *force_null_flags; /* per-column CSV FN flags */
- bool convert_selectively; /* do selective binary conversion? */
- CopyOnErrorChoice on_error; /* what to do when error happened */
- List *convert_select; /* list of column names (can be NIL) */
- CopyToRoutine *to_routine; /* callback routines for COPY TO */
-} CopyFormatOptions;
-
/* This is private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
-typedef void (*copy_data_dest_cb) (void *data, int len);
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 9c25e1c415..a869d78d72 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -14,10 +14,10 @@
#ifndef COPYAPI_H
#define COPYAPI_H
+#include "executor/execdesc.h"
#include "executor/tuptable.h"
#include "nodes/parsenodes.h"
-/* This is private in commands/copyto.c */
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
@@ -58,4 +58,122 @@ extern CopyToRoutine CopyToRoutineText;
extern CopyToRoutine CopyToRoutineCSV;
extern CopyToRoutine CopyToRoutineBinary;
+/*
+ * Represents whether a header line should be present, and whether it must
+ * match the actual names (which implies "true").
+ */
+typedef enum CopyHeaderChoice
+{
+ COPY_HEADER_FALSE = 0,
+ COPY_HEADER_TRUE,
+ COPY_HEADER_MATCH,
+} CopyHeaderChoice;
+
+/*
+ * Represents where to save input processing errors. More values to be added
+ * in the future.
+ */
+typedef enum CopyOnErrorChoice
+{
+ COPY_ON_ERROR_STOP = 0, /* immediately throw errors, default */
+ COPY_ON_ERROR_IGNORE, /* ignore errors */
+} CopyOnErrorChoice;
+
+/*
+ * A struct to hold COPY options, in a parsed form. All of these are related
+ * to formatting, except for 'freeze', which doesn't really belong here, but
+ * it's expedient to parse it along with all the other options.
+ */
+typedef struct CopyFormatOptions
+{
+ /* parameters from the COPY command */
+ int file_encoding; /* file or remote side's character encoding,
+ * -1 if not specified */
+ bool binary; /* binary format? */
+ bool freeze; /* freeze rows on loading? */
+ bool csv_mode; /* Comma Separated Value format? */
+ CopyHeaderChoice header_line; /* header line? */
+ char *null_print; /* NULL marker string (server encoding!) */
+ int null_print_len; /* length of same */
+ char *null_print_client; /* same converted to file encoding */
+ char *default_print; /* DEFAULT marker string */
+ int default_print_len; /* length of same */
+ char *delim; /* column delimiter (must be 1 byte) */
+ char *quote; /* CSV quote char (must be 1 byte) */
+ char *escape; /* CSV escape char (must be 1 byte) */
+ List *force_quote; /* list of column names */
+ bool force_quote_all; /* FORCE_QUOTE *? */
+ bool *force_quote_flags; /* per-column CSV FQ flags */
+ List *force_notnull; /* list of column names */
+ bool force_notnull_all; /* FORCE_NOT_NULL *? */
+ bool *force_notnull_flags; /* per-column CSV FNN flags */
+ List *force_null; /* list of column names */
+ bool force_null_all; /* FORCE_NULL *? */
+ bool *force_null_flags; /* per-column CSV FN flags */
+ bool convert_selectively; /* do selective binary conversion? */
+ CopyOnErrorChoice on_error; /* what to do when error happened */
+ List *convert_select; /* list of column names (can be NIL) */
+ CopyToRoutine *to_routine; /* callback routines for COPY TO */
+} CopyFormatOptions;
+
+/*
+ * Represents the different dest cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopyDest
+{
+ COPY_DEST_FILE, /* to file (or a piped program) */
+ COPY_DEST_FRONTEND, /* to frontend */
+ COPY_DEST_CALLBACK, /* to callback function */
+} CopyDest;
+
+typedef void (*copy_data_dest_cb) (void *data, int len);
+
+/*
+ * This struct contains all the state variables used throughout a COPY TO
+ * operation.
+ *
+ * Multi-byte encodings: all supported client-side encodings encode multi-byte
+ * characters by having the first byte's high bit set. Subsequent bytes of the
+ * character can have the high bit not set. When scanning data in such an
+ * encoding to look for a match to a single-byte (ie ASCII) character, we must
+ * use the full pg_encoding_mblen() machinery to skip over multibyte
+ * characters, else we might find a false match to a trailing byte. In
+ * supported server encodings, there is no possibility of a false match, and
+ * it's faster to make useless comparisons to trailing bytes than it is to
+ * invoke pg_encoding_mblen() to skip over them. encoding_embeds_ascii is true
+ * when we have to do it the hard way.
+ */
+typedef struct CopyToStateData
+{
+ /* low-level state data */
+ CopyDest copy_dest; /* type of copy source/destination */
+ FILE *copy_file; /* used if copy_dest == COPY_FILE */
+ StringInfo fe_msgbuf; /* used for all dests during COPY TO */
+
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy to */
+ QueryDesc *queryDesc; /* executable query to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDOUT */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_dest_cb data_dest_cb; /* function for writing data */
+
+ CopyFormatOptions opts;
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ FmgrInfo *out_functions; /* lookup info for output functions */
+ MemoryContext rowcontext; /* per-row evaluation context */
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyToStateData;
+
#endif /* COPYAPI_H */
--
2.41.0
[application/octet-stream] v8-0005-Extract-COPY-FROM-format-implementations.patch (24.8K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/6-v8-0005-Extract-COPY-FROM-format-implementations.patch)
download | inline diff:
From 781955f19ad27cdd66748be539bf45cf1b925856 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Tue, 23 Jan 2024 17:21:23 +0900
Subject: [PATCH v8 05/10] Extract COPY FROM format implementations
This doesn't change the current behavior. This just introduces
CopyFromRoutine, which just has function pointers of format
implementation like TupleTableSlotOps, and use it for existing "text",
"csv" and "binary" format implementations.
Note that CopyFromRoutine can't be used from extensions yet because
CopyRead*() aren't exported yet. Extensions can't read data from a
source without CopyRead*(). They will be exported by subsequent
patches.
---
src/backend/commands/copy.c | 3 +
src/backend/commands/copyfrom.c | 216 +++++++++++----
src/backend/commands/copyfromparse.c | 326 ++++++++++++-----------
src/include/commands/copy.h | 3 -
src/include/commands/copyapi.h | 44 +++
src/include/commands/copyfrom_internal.h | 4 +
6 files changed, 391 insertions(+), 205 deletions(-)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 6f0db0ae7c..ec6dfff8ab 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -459,12 +459,14 @@ ProcessCopyOptionCustomFormat(ParseState *pstate,
else if (strcmp(format, "csv") == 0)
{
opts_out->csv_mode = true;
+ opts_out->from_routine = &CopyFromRoutineCSV;
opts_out->to_routine = &CopyToRoutineCSV;
return;
}
else if (strcmp(format, "binary") == 0)
{
opts_out->binary = true;
+ opts_out->from_routine = &CopyFromRoutineBinary;
opts_out->to_routine = &CopyToRoutineBinary;
return;
}
@@ -533,6 +535,7 @@ ProcessCopyOptions(ParseState *pstate,
opts_out->file_encoding = -1;
/* Text is the default format. */
+ opts_out->from_routine = &CopyFromRoutineText;
opts_out->to_routine = &CopyToRoutineText;
/*
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index fb3d4d9296..d556ebb5d6 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -108,6 +108,170 @@ static char *limit_printout_length(const char *str);
static void ClosePipeFromProgram(CopyFromState cstate);
+
+/*
+ * CopyFromRoutine implementations.
+ */
+
+/*
+ * CopyFromRoutine implementation for "text" and "csv". CopyFromText*()
+ * refer cstate->opts.csv_mode and change their behavior. We can split this
+ * implementation and stop referring cstate->opts.csv_mode later.
+ */
+
+/* All "text" and "csv" options are parsed in ProcessCopyOptions(). We may
+ * move the code to here later. */
+static bool
+CopyFromTextProcessOption(CopyFromState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyFromTextGetFormat(CopyFromState cstate)
+{
+ return 0;
+}
+
+static void
+CopyFromTextStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ AttrNumber num_phys_attrs = tupDesc->natts;
+ AttrNumber attr_count;
+
+ /*
+ * If encoding conversion is needed, we need another buffer to hold the
+ * converted input data. Otherwise, we can just point input_buf to the
+ * same buffer as raw_buf.
+ */
+ if (cstate->need_transcoding)
+ {
+ cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
+ cstate->input_buf_index = cstate->input_buf_len = 0;
+ }
+ else
+ cstate->input_buf = cstate->raw_buf;
+ cstate->input_reached_eof = false;
+
+ initStringInfo(&cstate->line_buf);
+
+ /*
+ * Pick up the required catalog information for each attribute in the
+ * relation, including the input function, the element type (to pass to
+ * the input function).
+ */
+ cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
+ for (int attnum = 1; attnum <= num_phys_attrs; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
+ Oid in_func_oid;
+
+ /* We don't need info for dropped attributes */
+ if (att->attisdropped)
+ continue;
+
+ /* Fetch the input function and typioparam info */
+ getTypeInputInfo(att->atttypid,
+ &in_func_oid, &cstate->typioparams[attnum - 1]);
+ fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]);
+ }
+
+ /* create workspace for CopyReadAttributes results */
+ attr_count = list_length(cstate->attnumlist);
+ cstate->max_fields = attr_count;
+ cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
+}
+
+static void
+CopyFromTextEnd(CopyFromState cstate)
+{
+}
+
+/*
+ * CopyFromRoutine implementation for "binary".
+ */
+
+/* All "binary" options are parsed in ProcessCopyOptions(). We may move the
+ * code to here later. */
+static bool
+CopyFromBinaryProcessOption(CopyFromState cstate, DefElem *defel)
+{
+ return false;
+}
+
+static int16
+CopyFromBinaryGetFormat(CopyFromState cstate)
+{
+ return 1;
+}
+
+static void
+CopyFromBinaryStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ AttrNumber num_phys_attrs = tupDesc->natts;
+
+ /*
+ * Pick up the required catalog information for each attribute in the
+ * relation, including the input function, the element type (to pass to
+ * the input function).
+ */
+ cstate->in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
+ cstate->typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
+ for (int attnum = 1; attnum <= num_phys_attrs; attnum++)
+ {
+ Form_pg_attribute att = TupleDescAttr(tupDesc, attnum - 1);
+ Oid in_func_oid;
+
+ /* We don't need info for dropped attributes */
+ if (att->attisdropped)
+ continue;
+
+ /* Fetch the input function and typioparam info */
+ getTypeBinaryInputInfo(att->atttypid,
+ &in_func_oid, &cstate->typioparams[attnum - 1]);
+ fmgr_info(in_func_oid, &cstate->in_functions[attnum - 1]);
+ }
+
+ /* Read and verify binary header */
+ ReceiveCopyBinaryHeader(cstate);
+}
+
+static void
+CopyFromBinaryEnd(CopyFromState cstate)
+{
+}
+
+CopyFromRoutine CopyFromRoutineText = {
+ .CopyFromProcessOption = CopyFromTextProcessOption,
+ .CopyFromGetFormat = CopyFromTextGetFormat,
+ .CopyFromStart = CopyFromTextStart,
+ .CopyFromOneRow = CopyFromTextOneRow,
+ .CopyFromEnd = CopyFromTextEnd,
+};
+
+/*
+ * We can use the same CopyFromRoutine for both of "text" and "csv" because
+ * CopyFromText*() refer cstate->opts.csv_mode and change their behavior. We can
+ * split the implementations and stop referring cstate->opts.csv_mode later.
+ */
+CopyFromRoutine CopyFromRoutineCSV = {
+ .CopyFromProcessOption = CopyFromTextProcessOption,
+ .CopyFromGetFormat = CopyFromTextGetFormat,
+ .CopyFromStart = CopyFromTextStart,
+ .CopyFromOneRow = CopyFromTextOneRow,
+ .CopyFromEnd = CopyFromTextEnd,
+};
+
+CopyFromRoutine CopyFromRoutineBinary = {
+ .CopyFromProcessOption = CopyFromBinaryProcessOption,
+ .CopyFromGetFormat = CopyFromBinaryGetFormat,
+ .CopyFromStart = CopyFromBinaryStart,
+ .CopyFromOneRow = CopyFromBinaryOneRow,
+ .CopyFromEnd = CopyFromBinaryEnd,
+};
+
+
/*
* error context callback for COPY FROM
*
@@ -1384,9 +1548,6 @@ BeginCopyFrom(ParseState *pstate,
TupleDesc tupDesc;
AttrNumber num_phys_attrs,
num_defaults;
- FmgrInfo *in_functions;
- Oid *typioparams;
- Oid in_func_oid;
int *defmap;
ExprState **defexprs;
MemoryContext oldcontext;
@@ -1571,25 +1732,6 @@ BeginCopyFrom(ParseState *pstate,
cstate->raw_buf_index = cstate->raw_buf_len = 0;
cstate->raw_reached_eof = false;
- if (!cstate->opts.binary)
- {
- /*
- * If encoding conversion is needed, we need another buffer to hold
- * the converted input data. Otherwise, we can just point input_buf
- * to the same buffer as raw_buf.
- */
- if (cstate->need_transcoding)
- {
- cstate->input_buf = (char *) palloc(INPUT_BUF_SIZE + 1);
- cstate->input_buf_index = cstate->input_buf_len = 0;
- }
- else
- cstate->input_buf = cstate->raw_buf;
- cstate->input_reached_eof = false;
-
- initStringInfo(&cstate->line_buf);
- }
-
initStringInfo(&cstate->attribute_buf);
/* Assign range table and rteperminfos, we'll need them in CopyFrom. */
@@ -1608,8 +1750,6 @@ BeginCopyFrom(ParseState *pstate,
* the input function), and info about defaults and constraints. (Which
* input function we use depends on text/binary format choice.)
*/
- in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
- typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
defmap = (int *) palloc(num_phys_attrs * sizeof(int));
defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
@@ -1621,15 +1761,6 @@ BeginCopyFrom(ParseState *pstate,
if (att->attisdropped)
continue;
- /* Fetch the input function and typioparam info */
- if (cstate->opts.binary)
- getTypeBinaryInputInfo(att->atttypid,
- &in_func_oid, &typioparams[attnum - 1]);
- else
- getTypeInputInfo(att->atttypid,
- &in_func_oid, &typioparams[attnum - 1]);
- fmgr_info(in_func_oid, &in_functions[attnum - 1]);
-
/* Get default info if available */
defexprs[attnum - 1] = NULL;
@@ -1689,8 +1820,6 @@ BeginCopyFrom(ParseState *pstate,
cstate->bytes_processed = 0;
/* We keep those variables in cstate. */
- cstate->in_functions = in_functions;
- cstate->typioparams = typioparams;
cstate->defmap = defmap;
cstate->defexprs = defexprs;
cstate->volatile_defexprs = volatile_defexprs;
@@ -1763,20 +1892,7 @@ BeginCopyFrom(ParseState *pstate,
pgstat_progress_update_multi_param(3, progress_cols, progress_vals);
- if (cstate->opts.binary)
- {
- /* Read and verify binary header */
- ReceiveCopyBinaryHeader(cstate);
- }
-
- /* create workspace for CopyReadAttributes results */
- if (!cstate->opts.binary)
- {
- AttrNumber attr_count = list_length(cstate->attnumlist);
-
- cstate->max_fields = attr_count;
- cstate->raw_fields = (char **) palloc(attr_count * sizeof(char *));
- }
+ cstate->opts.from_routine->CopyFromStart(cstate, tupDesc);
MemoryContextSwitchTo(oldcontext);
@@ -1789,6 +1905,8 @@ BeginCopyFrom(ParseState *pstate,
void
EndCopyFrom(CopyFromState cstate)
{
+ cstate->opts.from_routine->CopyFromEnd(cstate);
+
/* No COPY FROM related resources except memory. */
if (cstate->is_program)
{
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 7cacd0b752..49632f75e4 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -172,7 +172,7 @@ ReceiveCopyBegin(CopyFromState cstate)
{
StringInfoData buf;
int natts = list_length(cstate->attnumlist);
- int16 format = (cstate->opts.binary ? 1 : 0);
+ int16 format = cstate->opts.from_routine->CopyFromGetFormat(cstate);
int i;
pq_beginmessage(&buf, PqMsg_CopyInResponse);
@@ -840,199 +840,219 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields)
return true;
}
-/*
- * Read next tuple from file for COPY FROM. Return false if no more tuples.
- *
- * 'econtext' is used to evaluate default expression for each column that is
- * either not read from the file or is using the DEFAULT option of COPY FROM.
- * It can be NULL when no default values are used, i.e. when all columns are
- * read from the file, and DEFAULT option is unset.
- *
- * 'values' and 'nulls' arrays must be the same length as columns of the
- * relation passed to BeginCopyFrom. This function fills the arrays.
- */
bool
-NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
- Datum *values, bool *nulls)
+CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
{
TupleDesc tupDesc;
- AttrNumber num_phys_attrs,
- attr_count,
- num_defaults = cstate->num_defaults;
+ AttrNumber attr_count;
FmgrInfo *in_functions = cstate->in_functions;
Oid *typioparams = cstate->typioparams;
- int i;
- int *defmap = cstate->defmap;
ExprState **defexprs = cstate->defexprs;
+ char **field_strings;
+ ListCell *cur;
+ int fldct;
+ int fieldno;
+ char *string;
tupDesc = RelationGetDescr(cstate->rel);
- num_phys_attrs = tupDesc->natts;
attr_count = list_length(cstate->attnumlist);
- /* Initialize all values for row to NULL */
- MemSet(values, 0, num_phys_attrs * sizeof(Datum));
- MemSet(nulls, true, num_phys_attrs * sizeof(bool));
- MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+ /* read raw fields in the next line */
+ if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
+ return false;
- if (!cstate->opts.binary)
- {
- char **field_strings;
- ListCell *cur;
- int fldct;
- int fieldno;
- char *string;
+ /* check for overflowing fields */
+ if (attr_count > 0 && fldct > attr_count)
+ ereport(ERROR,
+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ errmsg("extra data after last expected column")));
- /* read raw fields in the next line */
- if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
- return false;
+ fieldno = 0;
- /* check for overflowing fields */
- if (attr_count > 0 && fldct > attr_count)
+ /* Loop to read the user attributes on the line. */
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ int m = attnum - 1;
+ Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+ if (fieldno >= fldct)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("extra data after last expected column")));
+ errmsg("missing data for column \"%s\"",
+ NameStr(att->attname))));
+ string = field_strings[fieldno++];
- fieldno = 0;
-
- /* Loop to read the user attributes on the line. */
- foreach(cur, cstate->attnumlist)
+ if (cstate->convert_select_flags &&
+ !cstate->convert_select_flags[m])
{
- int attnum = lfirst_int(cur);
- int m = attnum - 1;
- Form_pg_attribute att = TupleDescAttr(tupDesc, m);
-
- if (fieldno >= fldct)
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("missing data for column \"%s\"",
- NameStr(att->attname))));
- string = field_strings[fieldno++];
-
- if (cstate->convert_select_flags &&
- !cstate->convert_select_flags[m])
- {
- /* ignore input field, leaving column as NULL */
- continue;
- }
+ /* ignore input field, leaving column as NULL */
+ continue;
+ }
- if (cstate->opts.csv_mode)
+ if (cstate->opts.csv_mode)
+ {
+ if (string == NULL &&
+ cstate->opts.force_notnull_flags[m])
{
- if (string == NULL &&
- cstate->opts.force_notnull_flags[m])
- {
- /*
- * FORCE_NOT_NULL option is set and column is NULL -
- * convert it to the NULL string.
- */
- string = cstate->opts.null_print;
- }
- else if (string != NULL && cstate->opts.force_null_flags[m]
- && strcmp(string, cstate->opts.null_print) == 0)
- {
- /*
- * FORCE_NULL option is set and column matches the NULL
- * string. It must have been quoted, or otherwise the
- * string would already have been set to NULL. Convert it
- * to NULL as specified.
- */
- string = NULL;
- }
+ /*
+ * FORCE_NOT_NULL option is set and column is NULL - convert
+ * it to the NULL string.
+ */
+ string = cstate->opts.null_print;
}
-
- cstate->cur_attname = NameStr(att->attname);
- cstate->cur_attval = string;
-
- if (string != NULL)
- nulls[m] = false;
-
- if (cstate->defaults[m])
+ else if (string != NULL && cstate->opts.force_null_flags[m]
+ && strcmp(string, cstate->opts.null_print) == 0)
{
/*
- * The caller must supply econtext and have switched into the
- * per-tuple memory context in it.
+ * FORCE_NULL option is set and column matches the NULL
+ * string. It must have been quoted, or otherwise the string
+ * would already have been set to NULL. Convert it to NULL as
+ * specified.
*/
- Assert(econtext != NULL);
- Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
-
- values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
+ string = NULL;
}
+ }
+
+ cstate->cur_attname = NameStr(att->attname);
+ cstate->cur_attval = string;
+
+ if (string != NULL)
+ nulls[m] = false;
+ if (cstate->defaults[m])
+ {
/*
- * If ON_ERROR is specified with IGNORE, skip rows with soft
- * errors
+ * The caller must supply econtext and have switched into the
+ * per-tuple memory context in it.
*/
- else if (!InputFunctionCallSafe(&in_functions[m],
- string,
- typioparams[m],
- att->atttypmod,
- (Node *) cstate->escontext,
- &values[m]))
- {
- cstate->num_errors++;
- return true;
- }
+ Assert(econtext != NULL);
+ Assert(CurrentMemoryContext == econtext->ecxt_per_tuple_memory);
- cstate->cur_attname = NULL;
- cstate->cur_attval = NULL;
+ values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]);
}
- Assert(fieldno == attr_count);
+ /*
+ * If ON_ERROR is specified with IGNORE, skip rows with soft errors
+ */
+ else if (!InputFunctionCallSafe(&in_functions[m],
+ string,
+ typioparams[m],
+ att->atttypmod,
+ (Node *) cstate->escontext,
+ &values[m]))
+ {
+ cstate->num_errors++;
+ return true;
+ }
+
+ cstate->cur_attname = NULL;
+ cstate->cur_attval = NULL;
}
- else
- {
- /* binary */
- int16 fld_count;
- ListCell *cur;
- cstate->cur_lineno++;
+ Assert(fieldno == attr_count);
- if (!CopyGetInt16(cstate, &fld_count))
- {
- /* EOF detected (end of file, or protocol-level EOF) */
- return false;
- }
+ return true;
+}
- if (fld_count == -1)
- {
- /*
- * Received EOF marker. Wait for the protocol-level EOF, and
- * complain if it doesn't come immediately. In COPY FROM STDIN,
- * this ensures that we correctly handle CopyFail, if client
- * chooses to send that now. When copying from file, we could
- * ignore the rest of the file like in text mode, but we choose to
- * be consistent with the COPY FROM STDIN case.
- */
- char dummy;
+bool
+CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
+{
+ TupleDesc tupDesc;
+ AttrNumber attr_count;
+ FmgrInfo *in_functions = cstate->in_functions;
+ Oid *typioparams = cstate->typioparams;
+ int16 fld_count;
+ ListCell *cur;
- if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("received copy data after EOF marker")));
- return false;
- }
+ tupDesc = RelationGetDescr(cstate->rel);
+ attr_count = list_length(cstate->attnumlist);
- if (fld_count != attr_count)
+ cstate->cur_lineno++;
+
+ if (!CopyGetInt16(cstate, &fld_count))
+ {
+ /* EOF detected (end of file, or protocol-level EOF) */
+ return false;
+ }
+
+ if (fld_count == -1)
+ {
+ /*
+ * Received EOF marker. Wait for the protocol-level EOF, and complain
+ * if it doesn't come immediately. In COPY FROM STDIN, this ensures
+ * that we correctly handle CopyFail, if client chooses to send that
+ * now. When copying from file, we could ignore the rest of the file
+ * like in text mode, but we choose to be consistent with the COPY
+ * FROM STDIN case.
+ */
+ char dummy;
+
+ if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("row field count is %d, expected %d",
- (int) fld_count, attr_count)));
+ errmsg("received copy data after EOF marker")));
+ return false;
+ }
- foreach(cur, cstate->attnumlist)
- {
- int attnum = lfirst_int(cur);
- int m = attnum - 1;
- Form_pg_attribute att = TupleDescAttr(tupDesc, m);
-
- cstate->cur_attname = NameStr(att->attname);
- values[m] = CopyReadBinaryAttribute(cstate,
- &in_functions[m],
- typioparams[m],
- att->atttypmod,
- &nulls[m]);
- cstate->cur_attname = NULL;
- }
+ if (fld_count != attr_count)
+ ereport(ERROR,
+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ errmsg("row field count is %d, expected %d",
+ (int) fld_count, attr_count)));
+
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ int m = attnum - 1;
+ Form_pg_attribute att = TupleDescAttr(tupDesc, m);
+
+ cstate->cur_attname = NameStr(att->attname);
+ values[m] = CopyReadBinaryAttribute(cstate,
+ &in_functions[m],
+ typioparams[m],
+ att->atttypmod,
+ &nulls[m]);
+ cstate->cur_attname = NULL;
}
+ return true;
+}
+
+/*
+ * Read next tuple from file for COPY FROM. Return false if no more tuples.
+ *
+ * 'econtext' is used to evaluate default expression for each column that is
+ * either not read from the file or is using the DEFAULT option of COPY FROM.
+ * It can be NULL when no default values are used, i.e. when all columns are
+ * read from the file, and DEFAULT option is unset.
+ *
+ * 'values' and 'nulls' arrays must be the same length as columns of the
+ * relation passed to BeginCopyFrom. This function fills the arrays.
+ */
+bool
+NextCopyFrom(CopyFromState cstate, ExprContext *econtext,
+ Datum *values, bool *nulls)
+{
+ TupleDesc tupDesc;
+ AttrNumber num_phys_attrs,
+ num_defaults = cstate->num_defaults;
+ int i;
+ int *defmap = cstate->defmap;
+ ExprState **defexprs = cstate->defexprs;
+
+ tupDesc = RelationGetDescr(cstate->rel);
+ num_phys_attrs = tupDesc->natts;
+
+ /* Initialize all values for row to NULL */
+ MemSet(values, 0, num_phys_attrs * sizeof(Datum));
+ MemSet(nulls, true, num_phys_attrs * sizeof(bool));
+ MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool));
+
+ if (!cstate->opts.from_routine->CopyFromOneRow(cstate, econtext, values,
+ nulls))
+ return false;
+
/*
* Now compute and insert any defaults available for the columns not
* provided by the input data. Anything not processed here or above will
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index b3f4682f95..df29d42555 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -20,9 +20,6 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
-/* This is private in commands/copyfrom.c */
-typedef struct CopyFromStateData *CopyFromState;
-
typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index ffad433a21..323e4705d2 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -18,6 +18,49 @@
#include "executor/tuptable.h"
#include "nodes/parsenodes.h"
+/* This is private in commands/copyfrom.c */
+typedef struct CopyFromStateData *CopyFromState;
+
+typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel);
+typedef int16 (*CopyFromGetFormat_function) (CopyFromState cstate);
+typedef void (*CopyFromStart_function) (CopyFromState cstate, TupleDesc tupDesc);
+typedef bool (*CopyFromOneRow_function) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
+typedef void (*CopyFromEnd_function) (CopyFromState cstate);
+
+/* Routines for a COPY FROM format implementation. */
+typedef struct CopyFromRoutine
+{
+ /*
+ * Called for processing one COPY FROM option. This will return false when
+ * the given option is invalid.
+ */
+ CopyFromProcessOption_function CopyFromProcessOption;
+
+ /*
+ * Called when COPY FROM is started. This will return a format as int16
+ * value. It's used for the CopyInResponse message.
+ */
+ CopyFromGetFormat_function CopyFromGetFormat;
+
+ /*
+ * Called when COPY FROM is started. This will initialize something and
+ * receive a header.
+ */
+ CopyFromStart_function CopyFromStart;
+
+ /* Copy one row. It returns false if no more tuples. */
+ CopyFromOneRow_function CopyFromOneRow;
+
+ /* Called when COPY FROM is ended. This will finalize something. */
+ CopyFromEnd_function CopyFromEnd;
+} CopyFromRoutine;
+
+/* Built-in CopyFromRoutine for "text", "csv" and "binary". */
+extern CopyFromRoutine CopyFromRoutineText;
+extern CopyFromRoutine CopyFromRoutineCSV;
+extern CopyFromRoutine CopyFromRoutineBinary;
+
+
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
@@ -113,6 +156,7 @@ typedef struct CopyFormatOptions
bool convert_selectively; /* do selective binary conversion? */
CopyOnErrorChoice on_error; /* what to do when error happened */
List *convert_select; /* list of column names (can be NIL) */
+ CopyFromRoutine *from_routine; /* callback routines for COPY FROM */
CopyToRoutine *to_routine; /* callback routines for COPY TO */
} CopyFormatOptions;
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index cad52fcc78..921c1513f7 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -183,4 +183,8 @@ typedef struct CopyFromStateData
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
+extern bool CopyFromTextOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
+extern bool CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
+
+
#endif /* COPYFROM_INTERNAL_H */
--
2.41.0
[application/octet-stream] v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch (7.6K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/7-v8-0009-change-CopyToGetFormat-to-CopyToSendCopyBegin-and.patch)
download | inline diff:
From f0a8151feff44823881c3c4e1e7aca4f9bd690d5 Mon Sep 17 00:00:00 2001
From: Zhao Junwang <[email protected]>
Date: Sat, 27 Jan 2024 09:53:31 +0800
Subject: [PATCH v8 09/10] change CopyToGetFormat to CopyToSendCopyBegin and
export more api
Signed-off-by: Zhao Junwang <[email protected]>
---
src/backend/commands/copyto.c | 65 ++++++++++---------
src/include/commands/copyapi.h | 12 ++--
.../test_copy_format/test_copy_format.c | 7 +-
3 files changed, 46 insertions(+), 38 deletions(-)
diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c
index b5d8678394..e2a4964015 100644
--- a/src/backend/commands/copyto.c
+++ b/src/backend/commands/copyto.c
@@ -66,11 +66,6 @@ static void CopyAttributeOutCSV(CopyToState cstate, const char *string,
/* Low-level communications functions */
static void SendCopyBegin(CopyToState cstate);
static void SendCopyEnd(CopyToState cstate);
-static void CopySendData(CopyToState cstate, const void *databuf, int datasize);
-static void CopySendString(CopyToState cstate, const char *str);
-static void CopySendChar(CopyToState cstate, char c);
-static void CopySendInt32(CopyToState cstate, int32 val);
-static void CopySendInt16(CopyToState cstate, int16 val);
/*
* CopyToRoutine implementations.
@@ -90,10 +85,20 @@ CopyToTextProcessOption(CopyToState cstate, DefElem *defel)
return false;
}
-static int16
-CopyToTextGetFormat(CopyToState cstate)
+static void
+CopyToTextSendCopyBegin(CopyToState cstate)
{
- return 0;
+ StringInfoData buf;
+ int natts = list_length(cstate->attnumlist);
+ int16 format = 0;
+ int i;
+
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
+ pq_sendbyte(&buf, format); /* overall format */
+ pq_sendint16(&buf, natts);
+ for (i = 0; i < natts; i++)
+ pq_sendint16(&buf, format); /* per-column formats */
+ pq_endmessage(&buf);
}
static void
@@ -230,10 +235,20 @@ CopyToBinaryProcessOption(CopyToState cstate, DefElem *defel)
return false;
}
-static int16
-CopyToBinaryGetFormat(CopyToState cstate)
+static void
+CopyToBinarySendCopyBegin(CopyToState cstate)
{
- return 1;
+ StringInfoData buf;
+ int natts = list_length(cstate->attnumlist);
+ int16 format = 1;
+ int i;
+
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
+ pq_sendbyte(&buf, format); /* overall format */
+ pq_sendint16(&buf, natts);
+ for (i = 0; i < natts; i++)
+ pq_sendint16(&buf, format); /* per-column formats */
+ pq_endmessage(&buf);
}
static void
@@ -315,7 +330,7 @@ CopyToBinaryEnd(CopyToState cstate)
CopyToRoutine CopyToRoutineText = {
.CopyToProcessOption = CopyToTextProcessOption,
- .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToSendCopyBegin = CopyToTextSendCopyBegin,
.CopyToStart = CopyToTextStart,
.CopyToOneRow = CopyToTextOneRow,
.CopyToEnd = CopyToTextEnd,
@@ -328,7 +343,7 @@ CopyToRoutine CopyToRoutineText = {
*/
CopyToRoutine CopyToRoutineCSV = {
.CopyToProcessOption = CopyToTextProcessOption,
- .CopyToGetFormat = CopyToTextGetFormat,
+ .CopyToSendCopyBegin = CopyToTextSendCopyBegin,
.CopyToStart = CopyToTextStart,
.CopyToOneRow = CopyToTextOneRow,
.CopyToEnd = CopyToTextEnd,
@@ -336,7 +351,7 @@ CopyToRoutine CopyToRoutineCSV = {
CopyToRoutine CopyToRoutineBinary = {
.CopyToProcessOption = CopyToBinaryProcessOption,
- .CopyToGetFormat = CopyToBinaryGetFormat,
+ .CopyToSendCopyBegin = CopyToBinarySendCopyBegin,
.CopyToStart = CopyToBinaryStart,
.CopyToOneRow = CopyToBinaryOneRow,
.CopyToEnd = CopyToBinaryEnd,
@@ -349,17 +364,7 @@ CopyToRoutine CopyToRoutineBinary = {
static void
SendCopyBegin(CopyToState cstate)
{
- StringInfoData buf;
- int natts = list_length(cstate->attnumlist);
- int16 format = cstate->opts.to_routine->CopyToGetFormat(cstate);
- int i;
-
- pq_beginmessage(&buf, PqMsg_CopyOutResponse);
- pq_sendbyte(&buf, format); /* overall format */
- pq_sendint16(&buf, natts);
- for (i = 0; i < natts; i++)
- pq_sendint16(&buf, format); /* per-column formats */
- pq_endmessage(&buf);
+ cstate->opts.to_routine->CopyToSendCopyBegin(cstate);
cstate->copy_dest = COPY_DEST_FRONTEND;
}
@@ -382,19 +387,19 @@ SendCopyEnd(CopyToState cstate)
* NB: no data conversion is applied by these functions
*----------
*/
-static void
+void
CopySendData(CopyToState cstate, const void *databuf, int datasize)
{
appendBinaryStringInfo(cstate->fe_msgbuf, databuf, datasize);
}
-static void
+void
CopySendString(CopyToState cstate, const char *str)
{
appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
}
-static void
+void
CopySendChar(CopyToState cstate, char c)
{
appendStringInfoCharMacro(cstate->fe_msgbuf, c);
@@ -464,7 +469,7 @@ CopyToStateFlush(CopyToState cstate)
/*
* CopySendInt32 sends an int32 in network byte order
*/
-static inline void
+inline void
CopySendInt32(CopyToState cstate, int32 val)
{
uint32 buf;
@@ -476,7 +481,7 @@ CopySendInt32(CopyToState cstate, int32 val)
/*
* CopySendInt16 sends an int16 in network byte order
*/
-static inline void
+inline void
CopySendInt16(CopyToState cstate, int16 val)
{
uint16 buf;
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 22accc83ab..0a05b24c54 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -67,7 +67,7 @@ extern CopyFromRoutine CopyFromRoutineBinary;
typedef struct CopyToStateData *CopyToState;
typedef bool (*CopyToProcessOption_function) (CopyToState cstate, DefElem *defel);
-typedef int16 (*CopyToGetFormat_function) (CopyToState cstate);
+typedef void (*CopyToSendCopyBegin_function) (CopyToState cstate);
typedef void (*CopyToStart_function) (CopyToState cstate, TupleDesc tupDesc);
typedef void (*CopyToOneRow_function) (CopyToState cstate, TupleTableSlot *slot);
typedef void (*CopyToEnd_function) (CopyToState cstate);
@@ -84,10 +84,9 @@ typedef struct CopyToRoutine
CopyToProcessOption_function CopyToProcessOption;
/*
- * Called when COPY TO is started. This will return a format as int16
- * value. It's used for the CopyOutResponse message.
+ * Called when COPY TO is started.
*/
- CopyToGetFormat_function CopyToGetFormat;
+ CopyToSendCopyBegin_function CopyToSendCopyBegin;
/* Called when COPY TO is started. This will send a header. */
CopyToStart_function CopyToStart;
@@ -384,6 +383,11 @@ typedef struct CopyToStateData
void *opaque; /* private space */
} CopyToStateData;
+extern void CopySendData(CopyToState cstate, const void *databuf, int datasize);
+extern void CopySendString(CopyToState cstate, const char *str);
+extern void CopySendChar(CopyToState cstate, char c);
+extern void CopySendInt32(CopyToState cstate, int32 val);
+extern void CopySendInt16(CopyToState cstate, int16 val);
extern void CopyToStateFlush(CopyToState cstate);
#endif /* COPYAPI_H */
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index 5e1b40e881..d833f22bbf 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -71,11 +71,10 @@ CopyToProcessOption(CopyToState cstate, DefElem *defel)
return true;
}
-static int16
-CopyToGetFormat(CopyToState cstate)
+static void
+CopyToSendCopyBegin(CopyToState cstate)
{
ereport(NOTICE, (errmsg("CopyToGetFormat")));
- return 0;
}
static void
@@ -99,7 +98,7 @@ CopyToEnd(CopyToState cstate)
static const CopyToRoutine CopyToRoutineTestCopyFormat = {
.type = T_CopyToRoutine,
.CopyToProcessOption = CopyToProcessOption,
- .CopyToGetFormat = CopyToGetFormat,
+ .CopyToSendCopyBegin = CopyToSendCopyBegin,
.CopyToStart = CopyToStart,
.CopyToOneRow = CopyToOneRow,
.CopyToEnd = CopyToEnd,
--
2.41.0
[application/octet-stream] v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch (7.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/8-v8-0006-Add-support-for-adding-custom-COPY-FROM-format.patch)
download | inline diff:
From f48e7b629a8d15fc70cd4cc4737dd2ad61910cc9 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Wed, 24 Jan 2024 11:07:14 +0900
Subject: [PATCH v8 06/10] Add support for adding custom COPY FROM format
We use the same approach as we used for custom COPY TO format. Now,
custom COPY format handler can return COPY TO format routines or COPY
FROM format routines based on the "is_from" argument:
copy_handler(true) returns CopyToRoutine
copy_handler(false) returns CopyFromRoutine
---
src/backend/commands/copy.c | 53 +++++++++++++------
src/include/commands/copyapi.h | 2 +
.../expected/test_copy_format.out | 12 +++++
.../test_copy_format/sql/test_copy_format.sql | 6 +++
.../test_copy_format/test_copy_format.c | 50 +++++++++++++++--
5 files changed, 105 insertions(+), 18 deletions(-)
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index ec6dfff8ab..479f36868c 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -472,12 +472,9 @@ ProcessCopyOptionCustomFormat(ParseState *pstate,
}
/* custom format */
- if (!is_from)
- {
- funcargtypes[0] = INTERNALOID;
- handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
- funcargtypes, true);
- }
+ funcargtypes[0] = INTERNALOID;
+ handlerOid = LookupFuncName(list_make1(makeString(format)), 1,
+ funcargtypes, true);
if (!OidIsValid(handlerOid))
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -486,14 +483,36 @@ ProcessCopyOptionCustomFormat(ParseState *pstate,
datum = OidFunctionCall1(handlerOid, BoolGetDatum(is_from));
routine = DatumGetPointer(datum);
- if (routine == NULL || !IsA(routine, CopyToRoutine))
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY handler function %s(%u) did not return a CopyToRoutine struct",
- format, handlerOid),
- parser_errposition(pstate, defel->location)));
-
- opts_out->to_routine = routine;
+ if (is_from)
+ {
+ if (routine == NULL || !IsA(routine, CopyFromRoutine))
+ ereport(
+ ERROR,
+ (errcode(
+ ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function "
+ "%s(%u) did not return a "
+ "CopyFromRoutine struct",
+ format, handlerOid),
+ parser_errposition(
+ pstate, defel->location)));
+ opts_out->from_routine = routine;
+ }
+ else
+ {
+ if (routine == NULL || !IsA(routine, CopyToRoutine))
+ ereport(
+ ERROR,
+ (errcode(
+ ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("COPY handler function "
+ "%s(%u) did not return a "
+ "CopyToRoutine struct",
+ format, handlerOid),
+ parser_errposition(
+ pstate, defel->location)));
+ opts_out->to_routine = routine;
+ }
}
/*
@@ -692,7 +711,11 @@ ProcessCopyOptions(ParseState *pstate,
{
bool processed = false;
- if (!is_from)
+ if (is_from)
+ processed =
+ opts_out->from_routine->CopyFromProcessOption(
+ cstate, defel);
+ else
processed = opts_out->to_routine->CopyToProcessOption(cstate, defel);
if (!processed)
ereport(ERROR,
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index 323e4705d2..ef1bb201c2 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -30,6 +30,8 @@ typedef void (*CopyFromEnd_function) (CopyFromState cstate);
/* Routines for a COPY FROM format implementation. */
typedef struct CopyFromRoutine
{
+ NodeTag type;
+
/*
* Called for processing one COPY FROM option. This will return false when
* the given option is invalid.
diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out
index 3a24ae7b97..6af69f0eb7 100644
--- a/src/test/modules/test_copy_format/expected/test_copy_format.out
+++ b/src/test/modules/test_copy_format/expected/test_copy_format.out
@@ -1,6 +1,18 @@
CREATE EXTENSION test_copy_format;
CREATE TABLE public.test (a INT, b INT, c INT);
INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test FROM stdin WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
+NOTICE: test_copy_format: is_from=true
+NOTICE: CopyFromProcessOption: "option_before"="before"
+NOTICE: CopyFromProcessOption: "option_after"="after"
+NOTICE: CopyFromGetFormat
+NOTICE: CopyFromStart: natts=3
+NOTICE: CopyFromOneRow
+NOTICE: CopyFromEnd
COPY public.test TO stdout WITH (
option_before 'before',
format 'test_copy_format',
diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql
index 0eb7ed2e11..94d3c789a0 100644
--- a/src/test/modules/test_copy_format/sql/test_copy_format.sql
+++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql
@@ -1,6 +1,12 @@
CREATE EXTENSION test_copy_format;
CREATE TABLE public.test (a INT, b INT, c INT);
INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789);
+COPY public.test FROM stdin WITH (
+ option_before 'before',
+ format 'test_copy_format',
+ option_after 'after'
+);
+\.
COPY public.test TO stdout WITH (
option_before 'before',
format 'test_copy_format',
diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c
index a2219afcde..5e1b40e881 100644
--- a/src/test/modules/test_copy_format/test_copy_format.c
+++ b/src/test/modules/test_copy_format/test_copy_format.c
@@ -18,6 +18,50 @@
PG_MODULE_MAGIC;
+static bool
+CopyFromProcessOption(CopyFromState cstate, DefElem *defel)
+{
+ ereport(NOTICE,
+ (errmsg("CopyFromProcessOption: \"%s\"=\"%s\"",
+ defel->defname, defGetString(defel))));
+ return true;
+}
+
+static int16
+CopyFromGetFormat(CopyFromState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyFromGetFormat")));
+ return 0;
+}
+
+static void
+CopyFromStart(CopyFromState cstate, TupleDesc tupDesc)
+{
+ ereport(NOTICE, (errmsg("CopyFromStart: natts=%d", tupDesc->natts)));
+}
+
+static bool
+CopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls)
+{
+ ereport(NOTICE, (errmsg("CopyFromOneRow")));
+ return false;
+}
+
+static void
+CopyFromEnd(CopyFromState cstate)
+{
+ ereport(NOTICE, (errmsg("CopyFromEnd")));
+}
+
+static const CopyFromRoutine CopyFromRoutineTestCopyFormat = {
+ .type = T_CopyFromRoutine,
+ .CopyFromProcessOption = CopyFromProcessOption,
+ .CopyFromGetFormat = CopyFromGetFormat,
+ .CopyFromStart = CopyFromStart,
+ .CopyFromOneRow = CopyFromOneRow,
+ .CopyFromEnd = CopyFromEnd,
+};
+
static bool
CopyToProcessOption(CopyToState cstate, DefElem *defel)
{
@@ -71,7 +115,7 @@ test_copy_format(PG_FUNCTION_ARGS)
(errmsg("test_copy_format: is_from=%s", is_from ? "true" : "false")));
if (is_from)
- elog(ERROR, "COPY FROM isn't supported yet");
-
- PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat);
+ PG_RETURN_POINTER(&CopyFromRoutineTestCopyFormat);
+ else
+ PG_RETURN_POINTER(&CopyToRoutineTestCopyFormat);
}
--
2.41.0
[application/octet-stream] v8-0010-introduce-contrib-pg_copy_json.patch (16.3K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/9-v8-0010-introduce-contrib-pg_copy_json.patch)
download | inline diff:
From 7dc6c1c798178f31728d048d4d528181626b3695 Mon Sep 17 00:00:00 2001
From: Zhao Junwang <[email protected]>
Date: Sat, 27 Jan 2024 13:34:38 +0800
Subject: [PATCH v8 10/10] introduce contrib/pg_copy_json
Signed-off-by: Zhao Junwang <[email protected]>
---
contrib/Makefile | 1 +
contrib/meson.build | 1 +
contrib/pg_copy_json/.gitignore | 4 +
contrib/pg_copy_json/Makefile | 23 ++
.../pg_copy_json/expected/pg_copy_json.out | 80 +++++++
contrib/pg_copy_json/meson.build | 34 +++
contrib/pg_copy_json/pg_copy_json--1.0.sql | 9 +
contrib/pg_copy_json/pg_copy_json.c | 218 ++++++++++++++++++
contrib/pg_copy_json/pg_copy_json.control | 5 +
contrib/pg_copy_json/sql/pg_copy_json.sql | 59 +++++
src/backend/utils/adt/json.c | 5 +-
src/include/utils/json.h | 2 +
12 files changed, 438 insertions(+), 3 deletions(-)
create mode 100644 contrib/pg_copy_json/.gitignore
create mode 100644 contrib/pg_copy_json/Makefile
create mode 100644 contrib/pg_copy_json/expected/pg_copy_json.out
create mode 100644 contrib/pg_copy_json/meson.build
create mode 100644 contrib/pg_copy_json/pg_copy_json--1.0.sql
create mode 100644 contrib/pg_copy_json/pg_copy_json.c
create mode 100644 contrib/pg_copy_json/pg_copy_json.control
create mode 100644 contrib/pg_copy_json/sql/pg_copy_json.sql
diff --git a/contrib/Makefile b/contrib/Makefile
index da4e2316a3..82cc496aa2 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -32,6 +32,7 @@ SUBDIRS = \
pageinspect \
passwordcheck \
pg_buffercache \
+ pg_copy_json \
pg_freespacemap \
pg_prewarm \
pg_stat_statements \
diff --git a/contrib/meson.build b/contrib/meson.build
index c12dc906ca..38933d15d1 100644
--- a/contrib/meson.build
+++ b/contrib/meson.build
@@ -45,6 +45,7 @@ subdir('oid2name')
subdir('pageinspect')
subdir('passwordcheck')
subdir('pg_buffercache')
+subdir('pg_copy_json')
subdir('pgcrypto')
subdir('pg_freespacemap')
subdir('pg_prewarm')
diff --git a/contrib/pg_copy_json/.gitignore b/contrib/pg_copy_json/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/contrib/pg_copy_json/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/contrib/pg_copy_json/Makefile b/contrib/pg_copy_json/Makefile
new file mode 100644
index 0000000000..b0a348d618
--- /dev/null
+++ b/contrib/pg_copy_json/Makefile
@@ -0,0 +1,23 @@
+# contrib/pg_copy_json//Makefile
+
+MODULE_big = pg_copy_json
+OBJS = \
+ $(WIN32RES) \
+ pg_copy_json.o
+PGFILEDESC = "pg_copy_json - COPY TO JSON (JavaScript Object Notation) format"
+
+EXTENSION = pg_copy_json
+DATA = pg_copy_json--1.0.sql
+
+REGRESS = test_copy_format
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/pg_copy_json
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/pg_copy_json/expected/pg_copy_json.out b/contrib/pg_copy_json/expected/pg_copy_json.out
new file mode 100644
index 0000000000..73633c2303
--- /dev/null
+++ b/contrib/pg_copy_json/expected/pg_copy_json.out
@@ -0,0 +1,80 @@
+--
+-- COPY TO JSON
+--
+CREATE EXTENSION pg_copy_json;
+-- test copying in JSON format with various styles
+-- of embedded line ending characters
+create temp table copytest (
+ style text,
+ test text,
+ filler int);
+insert into copytest values('DOS',E'abc\r\ndef',1);
+insert into copytest values('Unix',E'abc\ndef',2);
+insert into copytest values('Mac',E'abc\rdef',3);
+insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4);
+copy copytest to stdout with (format 'json');
+{"style":"DOS","test":"abc\r\ndef","filler":1}
+{"style":"Unix","test":"abc\ndef","filler":2}
+{"style":"Mac","test":"abc\rdef","filler":3}
+{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
+-- pg_copy_json do not support COPY FROM
+copy copytest from stdout with (format 'json');
+ERROR: cannot use JSON mode in COPY FROM
+-- test copying in JSON format with various styles
+-- of embedded escaped characters
+create temp table copyjsontest (
+ id bigserial,
+ f1 text,
+ f2 timestamptz);
+insert into copyjsontest
+ select g.i,
+ CASE WHEN g.i % 2 = 0 THEN
+ 'line with '' in it: ' || g.i::text
+ ELSE
+ 'line with " in it: ' || g.i::text
+ END,
+ 'Mon Feb 10 17:32:01 1997 PST'
+ from generate_series(1,5) as g(i);
+insert into copyjsontest (f1) values
+(E'aaa\"bbb'::text),
+(E'aaa\\bbb'::text),
+(E'aaa\/bbb'::text),
+(E'aaa\bbbb'::text),
+(E'aaa\fbbb'::text),
+(E'aaa\nbbb'::text),
+(E'aaa\rbbb'::text),
+(E'aaa\tbbb'::text);
+copy copyjsontest to stdout with (format 'json');
+{"id":1,"f1":"line with \" in it: 1","f2":"1997-02-10T17:32:01-08:00"}
+{"id":2,"f1":"line with ' in it: 2","f2":"1997-02-10T17:32:01-08:00"}
+{"id":3,"f1":"line with \" in it: 3","f2":"1997-02-10T17:32:01-08:00"}
+{"id":4,"f1":"line with ' in it: 4","f2":"1997-02-10T17:32:01-08:00"}
+{"id":5,"f1":"line with \" in it: 5","f2":"1997-02-10T17:32:01-08:00"}
+{"id":1,"f1":"aaa\"bbb","f2":null}
+{"id":2,"f1":"aaa\\bbb","f2":null}
+{"id":3,"f1":"aaa/bbb","f2":null}
+{"id":4,"f1":"aaa\bbbb","f2":null}
+{"id":5,"f1":"aaa\fbbb","f2":null}
+{"id":6,"f1":"aaa\nbbb","f2":null}
+{"id":7,"f1":"aaa\rbbb","f2":null}
+{"id":8,"f1":"aaa\tbbb","f2":null}
+-- test force array
+copy copytest to stdout (format 'json', force_array);
+[
+ {"style":"DOS","test":"abc\r\ndef","filler":1}
+,{"style":"Unix","test":"abc\ndef","filler":2}
+,{"style":"Mac","test":"abc\rdef","filler":3}
+,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
+]
+copy copytest to stdout (format 'json', force_array true);
+[
+ {"style":"DOS","test":"abc\r\ndef","filler":1}
+,{"style":"Unix","test":"abc\ndef","filler":2}
+,{"style":"Mac","test":"abc\rdef","filler":3}
+,{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
+]
+copy copytest to stdout (format 'json', force_array false);
+{"style":"DOS","test":"abc\r\ndef","filler":1}
+{"style":"Unix","test":"abc\ndef","filler":2}
+{"style":"Mac","test":"abc\rdef","filler":3}
+{"style":"esc\\ape","test":"a\\r\\\r\\\n\\nb","filler":4}
diff --git a/contrib/pg_copy_json/meson.build b/contrib/pg_copy_json/meson.build
new file mode 100644
index 0000000000..71f9338267
--- /dev/null
+++ b/contrib/pg_copy_json/meson.build
@@ -0,0 +1,34 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+pg_copy_json_sources = files(
+ 'pg_copy_json.c',
+)
+
+if host_system == 'windows'
+ pg_copy_json_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+ '--NAME', 'pg_copy_json',
+ '--FILEDESC', 'pg_copy_json - COPY TO JSON format',])
+endif
+
+pg_copy_json = shared_module('pg_copy_json',
+ pg_copy_json_sources,
+ kwargs: contrib_mod_args,
+)
+contrib_targets += pg_copy_json
+
+install_data(
+ 'pg_copy_json--1.0.sql',
+ 'pg_copy_json.control',
+ kwargs: contrib_data_args,
+)
+
+tests += {
+ 'name': 'pg_copy_json',
+ 'sd': meson.current_source_dir(),
+ 'bd': meson.current_build_dir(),
+ 'regress': {
+ 'sql': [
+ 'pg_copy_json',
+ ],
+ },
+}
diff --git a/contrib/pg_copy_json/pg_copy_json--1.0.sql b/contrib/pg_copy_json/pg_copy_json--1.0.sql
new file mode 100644
index 0000000000..d738a1e7e9
--- /dev/null
+++ b/contrib/pg_copy_json/pg_copy_json--1.0.sql
@@ -0,0 +1,9 @@
+/* contrib/pg_copy_json/copy_json--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION pg_copy_json" to load this file. \quit
+
+CREATE FUNCTION pg_catalog.json(internal)
+ RETURNS copy_handler
+ AS 'MODULE_PATHNAME', 'copy_json'
+ LANGUAGE C;
diff --git a/contrib/pg_copy_json/pg_copy_json.c b/contrib/pg_copy_json/pg_copy_json.c
new file mode 100644
index 0000000000..cbfdee8e8b
--- /dev/null
+++ b/contrib/pg_copy_json/pg_copy_json.c
@@ -0,0 +1,218 @@
+/*--------------------------------------------------------------------------
+ *
+ * pg_copy_json.c
+ * COPY TO JSON (JavaScript Object Notation) format.
+ *
+ * Portions Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * contrib/test_copy_format.c
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "commands/copy.h"
+#include "commands/defrem.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "libpq/pqformat.h"
+#include "utils/json.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct
+{
+ /*
+ * Force output of square brackets as array decorations at the beginning
+ * and end of output, with commas between the rows.
+ */
+ bool force_array;
+ bool force_array_specified;
+
+ /* need delimiter to start next json array element */
+ bool json_row_delim_needed;
+} CopyJsonData;
+
+static inline void
+InitCopyJsonData(CopyJsonData *p)
+{
+ Assert(p);
+ p->force_array = false;
+ p->force_array_specified = false;
+ p->json_row_delim_needed = false;
+}
+
+static void
+CopyToJsonSendEndOfRow(CopyToState cstate)
+{
+ switch (cstate->copy_dest)
+ {
+ case COPY_DEST_FILE:
+ /* Default line termination depends on platform */
+#ifndef WIN32
+ CopySendChar(cstate, '\n');
+#else
+ CopySendString(cstate, "\r\n");
+#endif
+ break;
+ case COPY_DEST_FRONTEND:
+ /* The FE/BE protocol uses \n as newline for all platforms */
+ CopySendChar(cstate, '\n');
+ break;
+ default:
+ break;
+ }
+ CopyToStateFlush(cstate);
+}
+
+static bool
+CopyToJsonProcessOption(CopyToState cstate, DefElem *defel)
+{
+ CopyJsonData *p;
+
+ if (cstate->opaque == NULL)
+ {
+ MemoryContext oldcontext;
+ oldcontext = MemoryContextSwitchTo(cstate->copycontext);
+ cstate->opaque = palloc0(sizeof(CopyJsonData));
+ MemoryContextSwitchTo(oldcontext);
+ InitCopyJsonData(cstate->opaque);
+ }
+
+ p = (CopyJsonData *)cstate->opaque;
+
+ if (strcmp(defel->defname, "force_array") == 0)
+ {
+ if (p->force_array_specified)
+ ereport(ERROR,
+ errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("CopyToJsonProcessOption: redundant options \"%s\"=\"%s\"",
+ defel->defname, defGetString(defel)));
+ p->force_array_specified = true;
+ p->force_array = defGetBoolean(defel);
+
+ return true;
+ }
+
+ return false;
+}
+
+static void
+CopyToJsonSendCopyBegin(CopyToState cstate)
+{
+ StringInfoData buf;
+ int16 format = 0;
+
+ pq_beginmessage(&buf, PqMsg_CopyOutResponse);
+ pq_sendbyte(&buf, format); /* overall format */
+ /*
+ * JSON mode is always one non-binary column
+ */
+ pq_sendint16(&buf, 1);
+ pq_sendint16(&buf, 0);
+ pq_endmessage(&buf);
+}
+
+static void
+CopyToJsonStart(CopyToState cstate, TupleDesc tupDesc)
+{
+ CopyJsonData *p;
+
+ if (cstate->opaque == NULL)
+ {
+ MemoryContext oldcontext;
+ oldcontext = MemoryContextSwitchTo(cstate->copycontext);
+ cstate->opaque = palloc0(sizeof(CopyJsonData));
+ MemoryContextSwitchTo(oldcontext);
+ InitCopyJsonData(cstate->opaque);
+ }
+
+ /* No need to alloc cstate->out_functions */
+
+ p = (CopyJsonData *)cstate->opaque;
+
+ /* If FORCE_ARRAY has been specified send the open bracket. */
+ if (p->force_array)
+ {
+ CopySendChar(cstate, '[');
+ CopyToJsonSendEndOfRow(cstate);
+ }
+}
+
+static void
+CopyToJsonOneRow(CopyToState cstate, TupleTableSlot *slot)
+{
+ Datum rowdata;
+ StringInfo result;
+ CopyJsonData *p;
+
+ Assert(cstate->opaque);
+ p = (CopyJsonData *)cstate->opaque;
+
+ if(!cstate->rel)
+ {
+ for (int i = 0; i < slot->tts_tupleDescriptor->natts; i++)
+ {
+ /* Flat-copy the attribute array */
+ memcpy(TupleDescAttr(slot->tts_tupleDescriptor, i),
+ TupleDescAttr(cstate->queryDesc->tupDesc, i),
+ 1 * sizeof(FormData_pg_attribute));
+ }
+ BlessTupleDesc(slot->tts_tupleDescriptor);
+ }
+ rowdata = ExecFetchSlotHeapTupleDatum(slot);
+ result = makeStringInfo();
+ composite_to_json(rowdata, result, false);
+
+ if (p->json_row_delim_needed)
+ CopySendChar(cstate, ',');
+ else if (p->force_array)
+ {
+ /* first row needs no delimiter */
+ CopySendChar(cstate, ' ');
+ p->json_row_delim_needed = true;
+ }
+ CopySendData(cstate, result->data, result->len);
+ CopyToJsonSendEndOfRow(cstate);
+}
+
+static void
+CopyToJsonEnd(CopyToState cstate)
+{
+ CopyJsonData *p;
+
+ Assert(cstate->opaque);
+ p = (CopyJsonData *)cstate->opaque;
+
+ /* If FORCE_ARRAY has been specified send the close bracket. */
+ if (p->force_array)
+ {
+ CopySendChar(cstate, ']');
+ CopyToJsonSendEndOfRow(cstate);
+ }
+}
+
+static const CopyToRoutine CopyToRoutineJson = {
+ .type = T_CopyToRoutine,
+ .CopyToProcessOption = CopyToJsonProcessOption,
+ .CopyToSendCopyBegin = CopyToJsonSendCopyBegin,
+ .CopyToStart = CopyToJsonStart,
+ .CopyToOneRow = CopyToJsonOneRow,
+ .CopyToEnd = CopyToJsonEnd,
+};
+
+PG_FUNCTION_INFO_V1(copy_json);
+Datum
+copy_json(PG_FUNCTION_ARGS)
+{
+ bool is_from = PG_GETARG_BOOL(0);
+
+ if (is_from)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot use JSON mode in COPY FROM")));
+
+ PG_RETURN_POINTER(&CopyToRoutineJson);
+}
diff --git a/contrib/pg_copy_json/pg_copy_json.control b/contrib/pg_copy_json/pg_copy_json.control
new file mode 100644
index 0000000000..90b0a74603
--- /dev/null
+++ b/contrib/pg_copy_json/pg_copy_json.control
@@ -0,0 +1,5 @@
+# pg_copy_json extension
+comment = 'COPY TO JSON format'
+default_version = '1.0'
+module_pathname = '$libdir/pg_copy_json'
+relocatable = true
diff --git a/contrib/pg_copy_json/sql/pg_copy_json.sql b/contrib/pg_copy_json/sql/pg_copy_json.sql
new file mode 100644
index 0000000000..73e7e514ac
--- /dev/null
+++ b/contrib/pg_copy_json/sql/pg_copy_json.sql
@@ -0,0 +1,59 @@
+--
+-- COPY TO JSON
+--
+
+CREATE EXTENSION pg_copy_json;
+
+-- test copying in JSON format with various styles
+-- of embedded line ending characters
+
+create temp table copytest (
+ style text,
+ test text,
+ filler int);
+
+insert into copytest values('DOS',E'abc\r\ndef',1);
+insert into copytest values('Unix',E'abc\ndef',2);
+insert into copytest values('Mac',E'abc\rdef',3);
+insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4);
+
+copy copytest to stdout with (format 'json');
+
+-- pg_copy_json do not support COPY FROM
+copy copytest from stdout with (format 'json');
+
+-- test copying in JSON format with various styles
+-- of embedded escaped characters
+
+create temp table copyjsontest (
+ id bigserial,
+ f1 text,
+ f2 timestamptz);
+
+insert into copyjsontest
+ select g.i,
+ CASE WHEN g.i % 2 = 0 THEN
+ 'line with '' in it: ' || g.i::text
+ ELSE
+ 'line with " in it: ' || g.i::text
+ END,
+ 'Mon Feb 10 17:32:01 1997 PST'
+ from generate_series(1,5) as g(i);
+
+insert into copyjsontest (f1) values
+(E'aaa\"bbb'::text),
+(E'aaa\\bbb'::text),
+(E'aaa\/bbb'::text),
+(E'aaa\bbbb'::text),
+(E'aaa\fbbb'::text),
+(E'aaa\nbbb'::text),
+(E'aaa\rbbb'::text),
+(E'aaa\tbbb'::text);
+
+copy copyjsontest to stdout with (format 'json');
+
+-- test force array
+
+copy copytest to stdout (format 'json', force_array);
+copy copytest to stdout (format 'json', force_array true);
+copy copytest to stdout (format 'json', force_array false);
diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c
index d719a61f16..fabd4e611e 100644
--- a/src/backend/utils/adt/json.c
+++ b/src/backend/utils/adt/json.c
@@ -83,8 +83,6 @@ typedef struct JsonAggState
JsonUniqueBuilderState unique_check;
} JsonAggState;
-static void composite_to_json(Datum composite, StringInfo result,
- bool use_line_feeds);
static void array_dim_to_json(StringInfo result, int dim, int ndims, int *dims,
Datum *vals, bool *nulls, int *valcount,
JsonTypeCategory tcategory, Oid outfuncoid,
@@ -507,8 +505,9 @@ array_to_json_internal(Datum array, StringInfo result, bool use_line_feeds)
/*
* Turn a composite / record into JSON.
+ * Exported so COPY TO can use it.
*/
-static void
+void
composite_to_json(Datum composite, StringInfo result, bool use_line_feeds)
{
HeapTupleHeader td;
diff --git a/src/include/utils/json.h b/src/include/utils/json.h
index 6d7f1b387d..d5631171ad 100644
--- a/src/include/utils/json.h
+++ b/src/include/utils/json.h
@@ -17,6 +17,8 @@
#include "lib/stringinfo.h"
/* functions in json.c */
+extern void composite_to_json(Datum composite, StringInfo result,
+ bool use_line_feeds);
extern void escape_json(StringInfo buf, const char *str);
extern char *JsonEncodeDateTime(char *buf, Datum value, Oid typid,
const int *tzp);
--
2.41.0
[application/octet-stream] v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch (4.5K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/10-v8-0008-Add-support-for-implementing-custom-COPY-FROM-for.patch)
download | inline diff:
From 3e847de1acb2fd6966ef01192204448711ca3d5e Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Wed, 24 Jan 2024 14:19:08 +0900
Subject: [PATCH v8 08/10] Add support for implementing custom COPY FROM format
as extension
* Add CopyFromStateData::opaque that can be used to keep data for
custom COPY From format implementation
* Export CopyReadBinaryData() to read the next data
* Rename CopyReadBinaryData() to CopyFromStateRead() because it's a
method for CopyFromState and "BinaryData" is redundant.
---
src/backend/commands/copyfromparse.c | 21 ++++++++++-----------
src/include/commands/copyapi.h | 5 +++++
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index a78a790060..f8a194635d 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -165,7 +165,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf,
static inline bool CopyGetInt32(CopyFromState cstate, int32 *val);
static inline bool CopyGetInt16(CopyFromState cstate, int16 *val);
static void CopyLoadInputBuf(CopyFromState cstate);
-static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes);
void
ReceiveCopyBegin(CopyFromState cstate)
@@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
int32 tmp;
/* Signature */
- if (CopyReadBinaryData(cstate, readSig, 11) != 11 ||
+ if (CopyFromStateRead(cstate, readSig, 11) != 11 ||
memcmp(readSig, BinarySignature, 11) != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
@@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate)
/* Skip extension header, if present */
while (tmp-- > 0)
{
- if (CopyReadBinaryData(cstate, readSig, 1) != 1)
+ if (CopyFromStateRead(cstate, readSig, 1) != 1)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("invalid COPY file header (wrong length)")));
@@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val)
{
uint32 buf;
- if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+ if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
{
*val = 0; /* suppress compiler warning */
return false;
@@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val)
{
uint16 buf;
- if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
+ if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf))
{
*val = 0; /* suppress compiler warning */
return false;
@@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate)
}
/*
- * CopyReadBinaryData
+ * CopyFromStateRead
*
* Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf
* and writes them to 'dest'. Returns the number of bytes read (which
* would be less than 'nbytes' only if we reach EOF).
*/
-static int
-CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes)
+int
+CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes)
{
int copied_bytes = 0;
@@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values,
*/
char dummy;
- if (CopyReadBinaryData(cstate, &dummy, 1) > 0)
+ if (CopyFromStateRead(cstate, &dummy, 1) > 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("received copy data after EOF marker")));
@@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo,
resetStringInfo(&cstate->attribute_buf);
enlargeStringInfo(&cstate->attribute_buf, fld_size);
- if (CopyReadBinaryData(cstate, cstate->attribute_buf.data,
- fld_size) != fld_size)
+ if (CopyFromStateRead(cstate, cstate->attribute_buf.data,
+ fld_size) != fld_size)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unexpected EOF in COPY data")));
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index b7e8f627bf..22accc83ab 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -314,8 +314,13 @@ typedef struct CopyFromStateData
#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
uint64 bytes_processed; /* number of bytes processed so far */
+
+ /* For custom format implementation */
+ void *opaque; /* private space */
} CopyFromStateData;
+extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes);
+
/*
* Represents the different dest cases we need to worry about at
* the bottom level
--
2.41.0
[application/octet-stream] v8-0007-Export-CopyFromStateData.patch (17.0K, ../../CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com/11-v8-0007-Export-CopyFromStateData.patch)
download | inline diff:
From 1ed575fda7f196ea411e9e53dd9c0739f160fb78 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <[email protected]>
Date: Wed, 24 Jan 2024 14:16:29 +0900
Subject: [PATCH v8 07/10] Export CopyFromStateData
It's for custom COPY FROM format handlers implemented as extension.
This just moves codes. This doesn't change codes except CopySource
enum values. CopySource enum values changes aren't required but I did
like I did for CopyDest enum values. I changed COPY_ prefix to
COPY_SOURCE_ prefix. For example, COPY_FILE to COPY_SOURCE_FILE.
Note that this change isn't enough to implement a custom COPY FROM
format handler as extension. We'll do the followings in a subsequent
commit:
1. Add an opaque space for custom COPY FROM format handler
2. Export CopyReadBinaryData() to read the next data
---
src/backend/commands/copyfrom.c | 4 +-
src/backend/commands/copyfromparse.c | 10 +-
src/include/commands/copy.h | 2 -
src/include/commands/copyapi.h | 156 ++++++++++++++++++++++-
src/include/commands/copyfrom_internal.h | 150 ----------------------
5 files changed, 162 insertions(+), 160 deletions(-)
diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c
index d556ebb5d6..b4ac7cbd2c 100644
--- a/src/backend/commands/copyfrom.c
+++ b/src/backend/commands/copyfrom.c
@@ -1710,7 +1710,7 @@ BeginCopyFrom(ParseState *pstate,
pg_encoding_to_char(GetDatabaseEncoding()))));
}
- cstate->copy_src = COPY_FILE; /* default */
+ cstate->copy_src = COPY_SOURCE_FILE; /* default */
cstate->whereClause = whereClause;
@@ -1829,7 +1829,7 @@ BeginCopyFrom(ParseState *pstate,
if (data_source_cb)
{
progress_vals[1] = PROGRESS_COPY_TYPE_CALLBACK;
- cstate->copy_src = COPY_CALLBACK;
+ cstate->copy_src = COPY_SOURCE_CALLBACK;
cstate->data_source_cb = data_source_cb;
}
else if (pipe)
diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c
index 49632f75e4..a78a790060 100644
--- a/src/backend/commands/copyfromparse.c
+++ b/src/backend/commands/copyfromparse.c
@@ -181,7 +181,7 @@ ReceiveCopyBegin(CopyFromState cstate)
for (i = 0; i < natts; i++)
pq_sendint16(&buf, format); /* per-column formats */
pq_endmessage(&buf);
- cstate->copy_src = COPY_FRONTEND;
+ cstate->copy_src = COPY_SOURCE_FRONTEND;
cstate->fe_msgbuf = makeStringInfo();
/* We *must* flush here to ensure FE knows it can send. */
pq_flush();
@@ -249,7 +249,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
switch (cstate->copy_src)
{
- case COPY_FILE:
+ case COPY_SOURCE_FILE:
bytesread = fread(databuf, 1, maxread, cstate->copy_file);
if (ferror(cstate->copy_file))
ereport(ERROR,
@@ -258,7 +258,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
if (bytesread == 0)
cstate->raw_reached_eof = true;
break;
- case COPY_FRONTEND:
+ case COPY_SOURCE_FRONTEND:
while (maxread > 0 && bytesread < minread && !cstate->raw_reached_eof)
{
int avail;
@@ -341,7 +341,7 @@ CopyGetData(CopyFromState cstate, void *databuf, int minread, int maxread)
bytesread += avail;
}
break;
- case COPY_CALLBACK:
+ case COPY_SOURCE_CALLBACK:
bytesread = cstate->data_source_cb(databuf, minread, maxread);
break;
}
@@ -1099,7 +1099,7 @@ CopyReadLine(CopyFromState cstate)
* after \. up to the protocol end of copy data. (XXX maybe better
* not to treat \. as special?)
*/
- if (cstate->copy_src == COPY_FRONTEND)
+ if (cstate->copy_src == COPY_SOURCE_FRONTEND)
{
int inbytes;
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index df29d42555..cd41d32074 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -20,8 +20,6 @@
#include "parser/parse_node.h"
#include "tcop/dest.h"
-typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
-
extern void DoCopy(ParseState *pstate, const CopyStmt *stmt,
int stmt_location, int stmt_len,
uint64 *processed);
diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h
index ef1bb201c2..b7e8f627bf 100644
--- a/src/include/commands/copyapi.h
+++ b/src/include/commands/copyapi.h
@@ -14,11 +14,12 @@
#ifndef COPYAPI_H
#define COPYAPI_H
+#include "commands/trigger.h"
#include "executor/execdesc.h"
#include "executor/tuptable.h"
+#include "nodes/miscnodes.h"
#include "nodes/parsenodes.h"
-/* This is private in commands/copyfrom.c */
typedef struct CopyFromStateData *CopyFromState;
typedef bool (*CopyFromProcessOption_function) (CopyFromState cstate, DefElem *defel);
@@ -162,6 +163,159 @@ typedef struct CopyFormatOptions
CopyToRoutine *to_routine; /* callback routines for COPY TO */
} CopyFormatOptions;
+
+/*
+ * Represents the different source cases we need to worry about at
+ * the bottom level
+ */
+typedef enum CopySource
+{
+ COPY_SOURCE_FILE, /* from file (or a piped program) */
+ COPY_SOURCE_FRONTEND, /* from frontend */
+ COPY_SOURCE_CALLBACK, /* from callback function */
+} CopySource;
+
+/*
+ * Represents the end-of-line terminator type of the input
+ */
+typedef enum EolType
+{
+ EOL_UNKNOWN,
+ EOL_NL,
+ EOL_CR,
+ EOL_CRNL,
+} EolType;
+
+typedef int (*copy_data_source_cb) (void *outbuf, int minread, int maxread);
+
+/*
+ * This struct contains all the state variables used throughout a COPY FROM
+ * operation.
+ */
+typedef struct CopyFromStateData
+{
+ /* low-level state data */
+ CopySource copy_src; /* type of copy source */
+ FILE *copy_file; /* used if copy_src == COPY_FILE */
+ StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
+
+ EolType eol_type; /* EOL type of input */
+ int file_encoding; /* file or remote side's character encoding */
+ bool need_transcoding; /* file encoding diff from server? */
+ Oid conversion_proc; /* encoding conversion function */
+
+ /* parameters from the COPY command */
+ Relation rel; /* relation to copy from */
+ List *attnumlist; /* integer list of attnums to copy */
+ char *filename; /* filename, or NULL for STDIN */
+ bool is_program; /* is 'filename' a program to popen? */
+ copy_data_source_cb data_source_cb; /* function for reading data */
+
+ CopyFormatOptions opts;
+ bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
+ Node *whereClause; /* WHERE condition (or NULL) */
+
+ /* these are just for error messages, see CopyFromErrorCallback */
+ const char *cur_relname; /* table name for error messages */
+ uint64 cur_lineno; /* line number for error messages */
+ const char *cur_attname; /* current att for error messages */
+ const char *cur_attval; /* current att value for error messages */
+ bool relname_only; /* don't output line number, att, etc. */
+
+ /*
+ * Working state
+ */
+ MemoryContext copycontext; /* per-copy execution context */
+
+ AttrNumber num_defaults; /* count of att that are missing and have
+ * default value */
+ FmgrInfo *in_functions; /* array of input functions for each attrs */
+ Oid *typioparams; /* array of element types for in_functions */
+ ErrorSaveContext *escontext; /* soft error trapper during in_functions
+ * execution */
+ uint64 num_errors; /* total number of rows which contained soft
+ * errors */
+ int *defmap; /* array of default att numbers related to
+ * missing att */
+ ExprState **defexprs; /* array of default att expressions for all
+ * att */
+ bool *defaults; /* if DEFAULT marker was found for
+ * corresponding att */
+ bool volatile_defexprs; /* is any of defexprs volatile? */
+ List *range_table; /* single element list of RangeTblEntry */
+ List *rteperminfos; /* single element list of RTEPermissionInfo */
+ ExprState *qualexpr;
+
+ TransitionCaptureState *transition_capture;
+
+ /*
+ * These variables are used to reduce overhead in COPY FROM.
+ *
+ * attribute_buf holds the separated, de-escaped text for each field of
+ * the current line. The CopyReadAttributes functions return arrays of
+ * pointers into this buffer. We avoid palloc/pfree overhead by re-using
+ * the buffer on each cycle.
+ *
+ * In binary COPY FROM, attribute_buf holds the binary data for the
+ * current field, but the usage is otherwise similar.
+ */
+ StringInfoData attribute_buf;
+
+ /* field raw data pointers found by COPY FROM */
+
+ int max_fields;
+ char **raw_fields;
+
+ /*
+ * Similarly, line_buf holds the whole input line being processed. The
+ * input cycle is first to read the whole line into line_buf, and then
+ * extract the individual attribute fields into attribute_buf. line_buf
+ * is preserved unmodified so that we can display it in error messages if
+ * appropriate. (In binary mode, line_buf is not used.)
+ */
+ StringInfoData line_buf;
+ bool line_buf_valid; /* contains the row being processed? */
+
+ /*
+ * input_buf holds input data, already converted to database encoding.
+ *
+ * In text mode, CopyReadLine parses this data sufficiently to locate line
+ * boundaries, then transfers the data to line_buf. We guarantee that
+ * there is a \0 at input_buf[input_buf_len] at all times. (In binary
+ * mode, input_buf is not used.)
+ *
+ * If encoding conversion is not required, input_buf is not a separate
+ * buffer but points directly to raw_buf. In that case, input_buf_len
+ * tracks the number of bytes that have been verified as valid in the
+ * database encoding, and raw_buf_len is the total number of bytes stored
+ * in the buffer.
+ */
+#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
+ char *input_buf;
+ int input_buf_index; /* next byte to process */
+ int input_buf_len; /* total # of bytes stored */
+ bool input_reached_eof; /* true if we reached EOF */
+ bool input_reached_error; /* true if a conversion error happened */
+ /* Shorthand for number of unconsumed bytes available in input_buf */
+#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
+
+ /*
+ * raw_buf holds raw input data read from the data source (file or client
+ * connection), not yet converted to the database encoding. Like with
+ * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
+ */
+#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
+ char *raw_buf;
+ int raw_buf_index; /* next byte to process */
+ int raw_buf_len; /* total # of bytes stored */
+ bool raw_reached_eof; /* true if we reached EOF */
+
+ /* Shorthand for number of unconsumed bytes available in raw_buf */
+#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
+
+ uint64 bytes_processed; /* number of bytes processed so far */
+} CopyFromStateData;
+
/*
* Represents the different dest cases we need to worry about at
* the bottom level
diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h
index 921c1513f7..f8f6120255 100644
--- a/src/include/commands/copyfrom_internal.h
+++ b/src/include/commands/copyfrom_internal.h
@@ -18,28 +18,6 @@
#include "commands/trigger.h"
#include "nodes/miscnodes.h"
-/*
- * Represents the different source cases we need to worry about at
- * the bottom level
- */
-typedef enum CopySource
-{
- COPY_FILE, /* from file (or a piped program) */
- COPY_FRONTEND, /* from frontend */
- COPY_CALLBACK, /* from callback function */
-} CopySource;
-
-/*
- * Represents the end-of-line terminator type of the input
- */
-typedef enum EolType
-{
- EOL_UNKNOWN,
- EOL_NL,
- EOL_CR,
- EOL_CRNL,
-} EolType;
-
/*
* Represents the insert method to be used during COPY FROM.
*/
@@ -52,134 +30,6 @@ typedef enum CopyInsertMethod
* ExecForeignBatchInsert only if valid */
} CopyInsertMethod;
-/*
- * This struct contains all the state variables used throughout a COPY FROM
- * operation.
- */
-typedef struct CopyFromStateData
-{
- /* low-level state data */
- CopySource copy_src; /* type of copy source */
- FILE *copy_file; /* used if copy_src == COPY_FILE */
- StringInfo fe_msgbuf; /* used if copy_src == COPY_FRONTEND */
-
- EolType eol_type; /* EOL type of input */
- int file_encoding; /* file or remote side's character encoding */
- bool need_transcoding; /* file encoding diff from server? */
- Oid conversion_proc; /* encoding conversion function */
-
- /* parameters from the COPY command */
- Relation rel; /* relation to copy from */
- List *attnumlist; /* integer list of attnums to copy */
- char *filename; /* filename, or NULL for STDIN */
- bool is_program; /* is 'filename' a program to popen? */
- copy_data_source_cb data_source_cb; /* function for reading data */
-
- CopyFormatOptions opts;
- bool *convert_select_flags; /* per-column CSV/TEXT CS flags */
- Node *whereClause; /* WHERE condition (or NULL) */
-
- /* these are just for error messages, see CopyFromErrorCallback */
- const char *cur_relname; /* table name for error messages */
- uint64 cur_lineno; /* line number for error messages */
- const char *cur_attname; /* current att for error messages */
- const char *cur_attval; /* current att value for error messages */
- bool relname_only; /* don't output line number, att, etc. */
-
- /*
- * Working state
- */
- MemoryContext copycontext; /* per-copy execution context */
-
- AttrNumber num_defaults; /* count of att that are missing and have
- * default value */
- FmgrInfo *in_functions; /* array of input functions for each attrs */
- Oid *typioparams; /* array of element types for in_functions */
- ErrorSaveContext *escontext; /* soft error trapper during in_functions
- * execution */
- uint64 num_errors; /* total number of rows which contained soft
- * errors */
- int *defmap; /* array of default att numbers related to
- * missing att */
- ExprState **defexprs; /* array of default att expressions for all
- * att */
- bool *defaults; /* if DEFAULT marker was found for
- * corresponding att */
- bool volatile_defexprs; /* is any of defexprs volatile? */
- List *range_table; /* single element list of RangeTblEntry */
- List *rteperminfos; /* single element list of RTEPermissionInfo */
- ExprState *qualexpr;
-
- TransitionCaptureState *transition_capture;
-
- /*
- * These variables are used to reduce overhead in COPY FROM.
- *
- * attribute_buf holds the separated, de-escaped text for each field of
- * the current line. The CopyReadAttributes functions return arrays of
- * pointers into this buffer. We avoid palloc/pfree overhead by re-using
- * the buffer on each cycle.
- *
- * In binary COPY FROM, attribute_buf holds the binary data for the
- * current field, but the usage is otherwise similar.
- */
- StringInfoData attribute_buf;
-
- /* field raw data pointers found by COPY FROM */
-
- int max_fields;
- char **raw_fields;
-
- /*
- * Similarly, line_buf holds the whole input line being processed. The
- * input cycle is first to read the whole line into line_buf, and then
- * extract the individual attribute fields into attribute_buf. line_buf
- * is preserved unmodified so that we can display it in error messages if
- * appropriate. (In binary mode, line_buf is not used.)
- */
- StringInfoData line_buf;
- bool line_buf_valid; /* contains the row being processed? */
-
- /*
- * input_buf holds input data, already converted to database encoding.
- *
- * In text mode, CopyReadLine parses this data sufficiently to locate line
- * boundaries, then transfers the data to line_buf. We guarantee that
- * there is a \0 at input_buf[input_buf_len] at all times. (In binary
- * mode, input_buf is not used.)
- *
- * If encoding conversion is not required, input_buf is not a separate
- * buffer but points directly to raw_buf. In that case, input_buf_len
- * tracks the number of bytes that have been verified as valid in the
- * database encoding, and raw_buf_len is the total number of bytes stored
- * in the buffer.
- */
-#define INPUT_BUF_SIZE 65536 /* we palloc INPUT_BUF_SIZE+1 bytes */
- char *input_buf;
- int input_buf_index; /* next byte to process */
- int input_buf_len; /* total # of bytes stored */
- bool input_reached_eof; /* true if we reached EOF */
- bool input_reached_error; /* true if a conversion error happened */
- /* Shorthand for number of unconsumed bytes available in input_buf */
-#define INPUT_BUF_BYTES(cstate) ((cstate)->input_buf_len - (cstate)->input_buf_index)
-
- /*
- * raw_buf holds raw input data read from the data source (file or client
- * connection), not yet converted to the database encoding. Like with
- * 'input_buf', we guarantee that there is a \0 at raw_buf[raw_buf_len].
- */
-#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
- char *raw_buf;
- int raw_buf_index; /* next byte to process */
- int raw_buf_len; /* total # of bytes stored */
- bool raw_reached_eof; /* true if we reached EOF */
-
- /* Shorthand for number of unconsumed bytes available in raw_buf */
-#define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index)
-
- uint64 bytes_processed; /* number of bytes processed so far */
-} CopyFromStateData;
-
extern void ReceiveCopyBegin(CopyFromState cstate);
extern void ReceiveCopyBinaryHeader(CopyFromState cstate);
--
2.41.0
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 06:03 ` Sutou Kouhei <[email protected]>
2024-01-29 06:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-29 06:03 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800,
Junwang Zhao <[email protected]> wrote:
> I have been working on a *COPY TO JSON* extension since yesterday,
> which is based on your V6 patch set, I'd like to give you more input
> so you can make better decisions about the implementation(with only
> pg-copy-arrow you might not get everything considered).
Thanks!
> 0009 is some changes made by me, I changed CopyToGetFormat to
> CopyToSendCopyBegin because pg_copy_json need to send different bytes
> in SendCopyBegin, get the format code along is not enough
Oh, I haven't cared about the case.
How about the following API instead?
static void
SendCopyBegin(CopyToState cstate)
{
StringInfoData buf;
pq_beginmessage(&buf, PqMsg_CopyOutResponse);
cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf);
pq_endmessage(&buf);
cstate->copy_dest = COPY_FRONTEND;
}
static void
CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf)
{
int16 format = 0;
pq_sendbyte(&buf, format); /* overall format */
/*
* JSON mode is always one non-binary column
*/
pq_sendint16(&buf, 1);
pq_sendint16(&buf, format);
}
> 00010 is the pg_copy_json extension, I think this should be a good
> case which can utilize the *extendable copy format* feature
It seems that it's convenient that we have one more callback
for initializing CopyToState::opaque. It's called only once
when Copy{To,From}Routine is chosen:
typedef struct CopyToRoutine
{
void (*CopyToInit) (CopyToState cstate);
...
};
void
ProcessCopyOptions(ParseState *pstate,
CopyFormatOptions *opts_out,
bool is_from,
void *cstate,
List *options)
{
...
foreach(option, options)
{
DefElem *defel = lfirst_node(DefElem, option);
if (strcmp(defel->defname, "format") == 0)
{
...
opts_out->to_routine = &CopyToRoutineXXX;
opts_out->to_routine->CopyToInit(cstate);
...
}
}
...
}
> maybe we
> should delete copy_test_format if we have this extension as an
> example?
I haven't read the COPY TO format json thread[1] carefully
(sorry), but we may add the JSON format as a built-in
format. If we do it, copy_test_format is useful to test the
extension API.
[1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m...
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-29 06:48 ` Junwang Zhao <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Junwang Zhao @ 2024-01-29 06:48 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 2:03 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3JDPks7XU5-NvzjzuKQYQqR8pDfS7CDGZonQTXfdWtnnw@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Sat, 27 Jan 2024 14:15:02 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> > I have been working on a *COPY TO JSON* extension since yesterday,
> > which is based on your V6 patch set, I'd like to give you more input
> > so you can make better decisions about the implementation(with only
> > pg-copy-arrow you might not get everything considered).
>
> Thanks!
>
> > 0009 is some changes made by me, I changed CopyToGetFormat to
> > CopyToSendCopyBegin because pg_copy_json need to send different bytes
> > in SendCopyBegin, get the format code along is not enough
>
> Oh, I haven't cared about the case.
> How about the following API instead?
>
> static void
> SendCopyBegin(CopyToState cstate)
> {
> StringInfoData buf;
>
> pq_beginmessage(&buf, PqMsg_CopyOutResponse);
> cstate->opts.to_routine->CopyToFillCopyOutResponse(cstate, &buf);
> pq_endmessage(&buf);
> cstate->copy_dest = COPY_FRONTEND;
> }
>
> static void
> CopyToJsonFillCopyOutResponse(CopyToState cstate, StringInfoData &buf)
> {
> int16 format = 0;
>
> pq_sendbyte(&buf, format); /* overall format */
> /*
> * JSON mode is always one non-binary column
> */
> pq_sendint16(&buf, 1);
> pq_sendint16(&buf, format);
> }
Make sense to me.
>
> > 00010 is the pg_copy_json extension, I think this should be a good
> > case which can utilize the *extendable copy format* feature
>
> It seems that it's convenient that we have one more callback
> for initializing CopyToState::opaque. It's called only once
> when Copy{To,From}Routine is chosen:
>
> typedef struct CopyToRoutine
> {
> void (*CopyToInit) (CopyToState cstate);
> ...
> };
I like this, we can alloc private data in this hook.
>
> void
> ProcessCopyOptions(ParseState *pstate,
> CopyFormatOptions *opts_out,
> bool is_from,
> void *cstate,
> List *options)
> {
> ...
> foreach(option, options)
> {
> DefElem *defel = lfirst_node(DefElem, option);
>
> if (strcmp(defel->defname, "format") == 0)
> {
> ...
> opts_out->to_routine = &CopyToRoutineXXX;
> opts_out->to_routine->CopyToInit(cstate);
> ...
> }
> }
> ...
> }
>
>
> > maybe we
> > should delete copy_test_format if we have this extension as an
> > example?
>
> I haven't read the COPY TO format json thread[1] carefully
> (sorry), but we may add the JSON format as a built-in
> format. If we do it, copy_test_format is useful to test the
> extension API.
Yeah, maybe, I have no strong opinion here, pg_copy_json is
just a toy extension for discussion.
>
> [1] https://www.postgresql.org/message-id/flat/CALvfUkBxTYy5uWPFVwpk_7ii2zgT07t3d-yR_cy4sfrrLU%3Dkcg%40m...
>
>
> Thanks,
> --
> kou
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 02:41 ` Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
1 sibling, 1 reply; 51+ messages in thread
From: Masahiko Sawada @ 2024-01-29 02:41 UTC (permalink / raw)
To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
>
> On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> >
> > Hi,
> >
> > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > Junwang Zhao <[email protected]> wrote:
> >
> > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > single option, and store the options in the opaque field, but it can not
> > > check the relation of two options, for example, considering json format,
> > > the `header` option can not be handled by these two functions.
> > >
> > > I want to find a way when the user specifies the header option, customer
> > > handler can error out.
> >
> > Ah, you want to use a built-in option (such as "header")
> > value from a custom handler, right? Hmm, it may be better
> > that we call CopyToProcessOption()/CopyFromProcessOption()
> > for all options including built-in options.
> >
> Hmm, still I don't think it can handle all cases, since we don't know
> the sequence of the options, we need all the options been parsed
> before we check the compatibility of the options, or customer
> handlers will need complicated logic to resolve that, which might
> lead to ugly code :(
>
Does it make sense to pass only non-builtin options to the custom
format callback after parsing and evaluating the builtin options? That
is, we parse and evaluate only the builtin options and populate
opts_out first, then pass each rest option to the custom format
handler callback. The callback can refer to the builtin option values.
The callback is expected to return false if the passed option is not
supported. If one of the builtin formats is specified and the rest
options list has at least one option, we raise "option %s not
recognized" error. IOW it's the core's responsibility to ranse the
"option %s not recognized" error, which is in order to raise a
consistent error message. Also, I think the core should check the
redundant options including bultiin and custom options.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-29 03:10 ` Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Junwang Zhao @ 2024-01-29 03:10 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote:
>
> On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
> >
> > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> > >
> > > Hi,
> > >
> > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > > Junwang Zhao <[email protected]> wrote:
> > >
> > > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > > single option, and store the options in the opaque field, but it can not
> > > > check the relation of two options, for example, considering json format,
> > > > the `header` option can not be handled by these two functions.
> > > >
> > > > I want to find a way when the user specifies the header option, customer
> > > > handler can error out.
> > >
> > > Ah, you want to use a built-in option (such as "header")
> > > value from a custom handler, right? Hmm, it may be better
> > > that we call CopyToProcessOption()/CopyFromProcessOption()
> > > for all options including built-in options.
> > >
> > Hmm, still I don't think it can handle all cases, since we don't know
> > the sequence of the options, we need all the options been parsed
> > before we check the compatibility of the options, or customer
> > handlers will need complicated logic to resolve that, which might
> > lead to ugly code :(
> >
>
> Does it make sense to pass only non-builtin options to the custom
> format callback after parsing and evaluating the builtin options? That
> is, we parse and evaluate only the builtin options and populate
> opts_out first, then pass each rest option to the custom format
> handler callback. The callback can refer to the builtin option values.
Yeah, I think this makes sense.
> The callback is expected to return false if the passed option is not
> supported. If one of the builtin formats is specified and the rest
> options list has at least one option, we raise "option %s not
> recognized" error. IOW it's the core's responsibility to ranse the
> "option %s not recognized" error, which is in order to raise a
> consistent error message. Also, I think the core should check the
> redundant options including bultiin and custom options.
It would be good that core could check all the redundant options,
but where should core do the book-keeping of all the options? I have
no idea about this, in my implementation of pg_copy_json extension,
I handle redundant options by adding a xxx_specified field for each
xxx.
>
> Regards,
>
> --
> Masahiko Sawada
> Amazon Web Services: https://aws.amazon.com
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 03:21 ` Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Masahiko Sawada @ 2024-01-29 03:21 UTC (permalink / raw)
To: Junwang Zhao <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote:
>
> On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote:
> >
> > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
> > >
> > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> > > >
> > > > Hi,
> > > >
> > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > > > Junwang Zhao <[email protected]> wrote:
> > > >
> > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > > > single option, and store the options in the opaque field, but it can not
> > > > > check the relation of two options, for example, considering json format,
> > > > > the `header` option can not be handled by these two functions.
> > > > >
> > > > > I want to find a way when the user specifies the header option, customer
> > > > > handler can error out.
> > > >
> > > > Ah, you want to use a built-in option (such as "header")
> > > > value from a custom handler, right? Hmm, it may be better
> > > > that we call CopyToProcessOption()/CopyFromProcessOption()
> > > > for all options including built-in options.
> > > >
> > > Hmm, still I don't think it can handle all cases, since we don't know
> > > the sequence of the options, we need all the options been parsed
> > > before we check the compatibility of the options, or customer
> > > handlers will need complicated logic to resolve that, which might
> > > lead to ugly code :(
> > >
> >
> > Does it make sense to pass only non-builtin options to the custom
> > format callback after parsing and evaluating the builtin options? That
> > is, we parse and evaluate only the builtin options and populate
> > opts_out first, then pass each rest option to the custom format
> > handler callback. The callback can refer to the builtin option values.
>
> Yeah, I think this makes sense.
>
> > The callback is expected to return false if the passed option is not
> > supported. If one of the builtin formats is specified and the rest
> > options list has at least one option, we raise "option %s not
> > recognized" error. IOW it's the core's responsibility to ranse the
> > "option %s not recognized" error, which is in order to raise a
> > consistent error message. Also, I think the core should check the
> > redundant options including bultiin and custom options.
>
> It would be good that core could check all the redundant options,
> but where should core do the book-keeping of all the options? I have
> no idea about this, in my implementation of pg_copy_json extension,
> I handle redundant options by adding a xxx_specified field for each
> xxx.
What I imagined is that while parsing the all specified options, we
evaluate builtin options and we add non-builtin options to another
list. Then when parsing a non-builtin option, we check if this option
already exists in the list. If there is, we raise the "option %s not
recognized" error.". Once we complete checking all options, we pass
each option in the list to the callback.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-29 03:37 ` Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Junwang Zhao @ 2024-01-29 03:37 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: Sutou Kouhei <[email protected]>; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 11:22 AM Masahiko Sawada <[email protected]> wrote:
>
> On Mon, Jan 29, 2024 at 12:10 PM Junwang Zhao <[email protected]> wrote:
> >
> > On Mon, Jan 29, 2024 at 10:42 AM Masahiko Sawada <[email protected]> wrote:
> > >
> > > On Fri, Jan 26, 2024 at 6:02 PM Junwang Zhao <[email protected]> wrote:
> > > >
> > > > On Fri, Jan 26, 2024 at 4:55 PM Sutou Kouhei <[email protected]> wrote:
> > > > >
> > > > > Hi,
> > > > >
> > > > > In <CAEG8a3KhS6s1XQgDSvc8vFTb4GkhBmS8TxOoVSDPFX+MPExxxQ@mail.gmail.com>
> > > > > "Re: Make COPY format extendable: Extract COPY TO format implementations" on Fri, 26 Jan 2024 16:41:50 +0800,
> > > > > Junwang Zhao <[email protected]> wrote:
> > > > >
> > > > > > CopyToProcessOption()/CopyFromProcessOption() can only handle
> > > > > > single option, and store the options in the opaque field, but it can not
> > > > > > check the relation of two options, for example, considering json format,
> > > > > > the `header` option can not be handled by these two functions.
> > > > > >
> > > > > > I want to find a way when the user specifies the header option, customer
> > > > > > handler can error out.
> > > > >
> > > > > Ah, you want to use a built-in option (such as "header")
> > > > > value from a custom handler, right? Hmm, it may be better
> > > > > that we call CopyToProcessOption()/CopyFromProcessOption()
> > > > > for all options including built-in options.
> > > > >
> > > > Hmm, still I don't think it can handle all cases, since we don't know
> > > > the sequence of the options, we need all the options been parsed
> > > > before we check the compatibility of the options, or customer
> > > > handlers will need complicated logic to resolve that, which might
> > > > lead to ugly code :(
> > > >
> > >
> > > Does it make sense to pass only non-builtin options to the custom
> > > format callback after parsing and evaluating the builtin options? That
> > > is, we parse and evaluate only the builtin options and populate
> > > opts_out first, then pass each rest option to the custom format
> > > handler callback. The callback can refer to the builtin option values.
> >
> > Yeah, I think this makes sense.
> >
> > > The callback is expected to return false if the passed option is not
> > > supported. If one of the builtin formats is specified and the rest
> > > options list has at least one option, we raise "option %s not
> > > recognized" error. IOW it's the core's responsibility to ranse the
> > > "option %s not recognized" error, which is in order to raise a
> > > consistent error message. Also, I think the core should check the
> > > redundant options including bultiin and custom options.
> >
> > It would be good that core could check all the redundant options,
> > but where should core do the book-keeping of all the options? I have
> > no idea about this, in my implementation of pg_copy_json extension,
> > I handle redundant options by adding a xxx_specified field for each
> > xxx.
>
> What I imagined is that while parsing the all specified options, we
> evaluate builtin options and we add non-builtin options to another
> list. Then when parsing a non-builtin option, we check if this option
> already exists in the list. If there is, we raise the "option %s not
> recognized" error.". Once we complete checking all options, we pass
> each option in the list to the callback.
LGTM.
>
> Regards,
>
> --
> Masahiko Sawada
> Amazon Web Services: https://aws.amazon.com
--
Regards
Junwang Zhao
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
@ 2024-01-29 09:45 ` Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-29 09:45 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800,
Junwang Zhao <[email protected]> wrote:
>> > > Does it make sense to pass only non-builtin options to the custom
>> > > format callback after parsing and evaluating the builtin options? That
>> > > is, we parse and evaluate only the builtin options and populate
>> > > opts_out first, then pass each rest option to the custom format
>> > > handler callback. The callback can refer to the builtin option values.
>>
>> What I imagined is that while parsing the all specified options, we
>> evaluate builtin options and we add non-builtin options to another
>> list. Then when parsing a non-builtin option, we check if this option
>> already exists in the list. If there is, we raise the "option %s not
>> recognized" error.". Once we complete checking all options, we pass
>> each option in the list to the callback.
I implemented this idea and the following ideas:
1. Add init callback for initialization
2. Change GetFormat() to FillCopyXXXResponse()
because JSON format always use 1 column
3. FROM only: Eliminate more cstate->opts.csv_mode branches
(This is for performance.)
See the attached v9 patch set for details. Changes since v7:
0001:
* Move CopyToProcessOption() calls to the end of
ProcessCopyOptions() for easy to option validation
* Add CopyToState::CopyToInit() and call it in
ProcessCopyOptionFormatTo()
* Change CopyToState::CopyToGetFormat() to
CopyToState::CopyToFillCopyOutResponse() and use it in
SendCopyBegin()
0002:
* Move CopyFromProcessOption() calls to the end of
ProcessCopyOptions() for easy to option validation
* Add CopyFromState::CopyFromInit() and call it in
ProcessCopyOptionFormatFrom()
* Change CopyFromState::CopyFromGetFormat() to
CopyFromState::CopyFromFillCopyOutResponse() and use it in
ReceiveCopyBegin()
* Rename NextCopyFromRawFields() to
NextCopyFromRawFieldsInternal() and pass the read
attributes callback explicitly to eliminate more
cstate->opts.csv_mode branches
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-30 02:11 ` Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Masahiko Sawada @ 2024-01-30 02:11 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Mon, Jan 29, 2024 at 6:45 PM Sutou Kouhei <[email protected]> wrote:
>
> Hi,
>
> In <CAEG8a3Jnmbjw82OiSvRK3v9XN2zSshsB8ew1mZCQDAkKq6r9YQ@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Mon, 29 Jan 2024 11:37:07 +0800,
> Junwang Zhao <[email protected]> wrote:
>
> >> > > Does it make sense to pass only non-builtin options to the custom
> >> > > format callback after parsing and evaluating the builtin options? That
> >> > > is, we parse and evaluate only the builtin options and populate
> >> > > opts_out first, then pass each rest option to the custom format
> >> > > handler callback. The callback can refer to the builtin option values.
> >>
> >> What I imagined is that while parsing the all specified options, we
> >> evaluate builtin options and we add non-builtin options to another
> >> list. Then when parsing a non-builtin option, we check if this option
> >> already exists in the list. If there is, we raise the "option %s not
> >> recognized" error.". Once we complete checking all options, we pass
> >> each option in the list to the callback.
>
> I implemented this idea and the following ideas:
>
> 1. Add init callback for initialization
> 2. Change GetFormat() to FillCopyXXXResponse()
> because JSON format always use 1 column
> 3. FROM only: Eliminate more cstate->opts.csv_mode branches
> (This is for performance.)
>
> See the attached v9 patch set for details. Changes since v7:
>
> 0001:
>
> * Move CopyToProcessOption() calls to the end of
> ProcessCopyOptions() for easy to option validation
> * Add CopyToState::CopyToInit() and call it in
> ProcessCopyOptionFormatTo()
> * Change CopyToState::CopyToGetFormat() to
> CopyToState::CopyToFillCopyOutResponse() and use it in
> SendCopyBegin()
Thank you for updating the patch! Here are comments on 0001 patch:
---
+ if (!format_specified)
+ /* Set the default format. */
+ ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
+
I think we can pass "text" in this case instead of NULL. That way,
ProcessCopyOptionFormatTo doesn't need to handle NULL case.
We need curly brackets for this "if branch" as follows:
if (!format_specifed)
{
/* Set the default format. */
ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
}
---
+ /* Process not built-in options. */
+ foreach(option, unknown_options)
+ {
+ DefElem *defel = lfirst_node(DefElem, option);
+ bool processed = false;
+
+ if (!is_from)
+ processed =
opts_out->to_routine->CopyToProcessOption(cstate, defel);
+ if (!processed)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("option \"%s\" not recognized",
+ defel->defname),
+ parser_errposition(pstate,
defel->location)));
+ }
+ list_free(unknown_options);
I think we can check the duplicated options in the core as we discussed.
---
+static void
+CopyToTextBasedInit(CopyToState cstate)
+{
+}
and
+static void
+CopyToBinaryInit(CopyToState cstate)
+{
+}
Do we really need separate callbacks for the same behavior? I think we
can have a common init function say CopyToBuitinInit() that does
nothing. Or we can make the init callback optional.
The same is true for process-option callback.
---
List *convert_select; /* list of column names (can be NIL) */
+ const CopyToRoutine *to_routine; /* callback
routines for COPY TO */
} CopyFormatOptions;
I think CopyToStateData is a better place to have CopyToRoutine.
copy_data_dest_cb is also there.
---
- if (strcmp(fmt, "text") == 0)
- /* default format */ ;
- else if (strcmp(fmt, "csv") == 0)
- opts_out->csv_mode = true;
- else if (strcmp(fmt, "binary") == 0)
- opts_out->binary = true;
+
+ if (is_from)
+ {
+ char *fmt = defGetString(defel);
+
+ if (strcmp(fmt, "text") == 0)
+ /* default format */ ;
+ else if (strcmp(fmt, "csv") == 0)
+ {
+ opts_out->csv_mode = true;
+ }
+ else if (strcmp(fmt, "binary") == 0)
+ {
+ opts_out->binary = true;
+ }
else
- ereport(ERROR,
-
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("COPY format
\"%s\" not recognized", fmt\),
-
parser_errposition(pstate, defel->location)));
+ ProcessCopyOptionFormatTo(pstate,
opts_out, cstate, defel);
The 0002 patch replaces the options checks with
ProcessCopyOptionFormatFrom(). However, both
ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would
set format-related options such as opts_out->csv_mode etc, which seems
not elegant. IIUC the reason why we process only the "format" option
first is to set the callback functions and call the init callback. So
I think we don't necessarily need to do both setting callbacks and
setting format-related options together. Probably we can do only the
callback stuff first and then set format-related options in the
original place we used to do?
---
+static void
+CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf)
+{
+ int16 format = 0;
+ int natts = list_length(cstate->attnumlist);
+ int i;
+
+ pq_sendbyte(buf, format); /* overall format */
+ pq_sendint16(buf, natts);
+ for (i = 0; i < natts; i++)
+ pq_sendint16(buf, format); /* per-column formats */
+}
This function and CopyToBinaryFillCopyOutResponse() fill three things:
overall format, the number of columns, and per-column formats. While
this approach is flexible, extensions will have to understand the
format of CopyOutResponse message. An alternative is to have one or
more callbacks that return these three things.
---
+ /* Get info about the columns we need to process. */
+ cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs *
sizeof(Fmgr\Info));
+ foreach(cur, cstate->attnumlist)
+ {
+ int attnum = lfirst_int(cur);
+ Oid out_func_oid;
+ bool isvarlena;
+ Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
+
+ getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
+ fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
+ }
Is preparing the out functions an extension's responsibility? I
thought the core could prepare them based on the overall format
specified by extensions, as long as the overall format matches the
actual data format to send. What do you think?
---
+ /*
+ * Called when COPY TO via the PostgreSQL protocol is
started. This must
+ * fill buf as a valid CopyOutResponse message:
+ *
+ */
+ /*--
+ * +--------+--------+--------+--------+--------+ +--------+--------+
+ * | Format | N attributes | Attr1's format |...| AttrN's format |
+ * +--------+--------+--------+--------+--------+ +--------+--------+
+ * 0: text 0: text 0: text
+ * 1: binary 1: binary 1: binary
+ */
I think this kind of diagram could be missed from being updated when
we update the CopyOutResponse format. It's better to refer to the
documentation instead.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
@ 2024-01-30 05:45 ` Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-30 05:45 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900,
Masahiko Sawada <[email protected]> wrote:
> ---
> + if (!format_specified)
> + /* Set the default format. */
> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
> +
>
> I think we can pass "text" in this case instead of NULL. That way,
> ProcessCopyOptionFormatTo doesn't need to handle NULL case.
Yes, we can do it. But it needs a DefElem allocation. Is it
acceptable?
> We need curly brackets for this "if branch" as follows:
>
> if (!format_specifed)
> {
> /* Set the default format. */
> ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
> }
Oh, sorry. I assumed that pgindent adjusts the style too.
> ---
> + /* Process not built-in options. */
> + foreach(option, unknown_options)
> + {
> + DefElem *defel = lfirst_node(DefElem, option);
> + bool processed = false;
> +
> + if (!is_from)
> + processed =
> opts_out->to_routine->CopyToProcessOption(cstate, defel);
> + if (!processed)
> + ereport(ERROR,
> + (errcode(ERRCODE_SYNTAX_ERROR),
> + errmsg("option \"%s\" not recognized",
> + defel->defname),
> + parser_errposition(pstate,
> defel->location)));
> + }
> + list_free(unknown_options);
>
> I think we can check the duplicated options in the core as we discussed.
Oh, sorry. I missed the part. I'll implement it.
> ---
> +static void
> +CopyToTextBasedInit(CopyToState cstate)
> +{
> +}
>
> and
>
> +static void
> +CopyToBinaryInit(CopyToState cstate)
> +{
> +}
>
> Do we really need separate callbacks for the same behavior? I think we
> can have a common init function say CopyToBuitinInit() that does
> nothing. Or we can make the init callback optional.
>
> The same is true for process-option callback.
OK. I'll make them optional.
> ---
> List *convert_select; /* list of column names (can be NIL) */
> + const CopyToRoutine *to_routine; /* callback
> routines for COPY TO */
> } CopyFormatOptions;
>
> I think CopyToStateData is a better place to have CopyToRoutine.
> copy_data_dest_cb is also there.
We can do it but ProcessCopyOptions() accepts NULL
CopyToState for file_fdw. Can we create an empty
CopyToStateData internally like we did for opts_out in
ProcessCopyOptions()? (But it requires exporting
CopyToStateData. We'll export it in a later patch but it's
not yet in 0001.)
> The 0002 patch replaces the options checks with
> ProcessCopyOptionFormatFrom(). However, both
> ProcessCopyOptionFormatTo() and ProcessCOpyOptionFormatFrom() would
> set format-related options such as opts_out->csv_mode etc, which seems
> not elegant. IIUC the reason why we process only the "format" option
> first is to set the callback functions and call the init callback. So
> I think we don't necessarily need to do both setting callbacks and
> setting format-related options together. Probably we can do only the
> callback stuff first and then set format-related options in the
> original place we used to do?
If we do it, we need to write the (strcmp(format, "csv") ==
0) condition in copyto.c and copy.c. I wanted to avoid it. I
think that the duplication (setting opts_out->csv_mode in
copyto.c and copyfrom.c) is not a problem. But it's not a
strong opinion. If (strcmp(format, "csv") == 0) duplication
is better than opts_out->csv_mode = true duplication, I'll
do it.
BTW, if we want to make the CSV format implementation more
modularized, we will remove opts_out->csv_mode, move CSV
related options to CopyToCSVProcessOption() and keep CSV
related options in its opaque space. For example,
opts_out->force_quote exists in COPY TO opaque space but
doesn't exist in COPY FROM opaque space because it's not
used in COPY FROM.
> +static void
> +CopyToTextBasedFillCopyOutResponse(CopyToState cstate, StringInfoData *buf)
> +{
> + int16 format = 0;
> + int natts = list_length(cstate->attnumlist);
> + int i;
> +
> + pq_sendbyte(buf, format); /* overall format */
> + pq_sendint16(buf, natts);
> + for (i = 0; i < natts; i++)
> + pq_sendint16(buf, format); /* per-column formats */
> +}
>
> This function and CopyToBinaryFillCopyOutResponse() fill three things:
> overall format, the number of columns, and per-column formats. While
> this approach is flexible, extensions will have to understand the
> format of CopyOutResponse message. An alternative is to have one or
> more callbacks that return these three things.
Yes, we can choose the approach. I don't have a strong
opinion on which approach to choose.
> + /* Get info about the columns we need to process. */
> + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs *
> sizeof(Fmgr\Info));
> + foreach(cur, cstate->attnumlist)
> + {
> + int attnum = lfirst_int(cur);
> + Oid out_func_oid;
> + bool isvarlena;
> + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
> +
> + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
> + }
>
> Is preparing the out functions an extension's responsibility? I
> thought the core could prepare them based on the overall format
> specified by extensions, as long as the overall format matches the
> actual data format to send. What do you think?
Hmm. I want to keep the preparation as an extension's
responsibility. Because it's not needed for all formats. For
example, Apache Arrow FORMAT doesn't need it. And JSON
FORMAT doesn't need it too because it use
composite_to_json().
> + /*
> + * Called when COPY TO via the PostgreSQL protocol is
> started. This must
> + * fill buf as a valid CopyOutResponse message:
> + *
> + */
> + /*--
> + * +--------+--------+--------+--------+--------+ +--------+--------+
> + * | Format | N attributes | Attr1's format |...| AttrN's format |
> + * +--------+--------+--------+--------+--------+ +--------+--------+
> + * 0: text 0: text 0: text
> + * 1: binary 1: binary 1: binary
> + */
>
> I think this kind of diagram could be missed from being updated when
> we update the CopyOutResponse format. It's better to refer to the
> documentation instead.
It makes sense. I couldn't find the documentation when I
wrote it but I found it now...:
https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY
Is there recommended comment style to refer a documentation?
"See doc/src/sgml/protocol.sgml for the CopyOutResponse
message details" is OK?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-30 07:20 ` Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Michael Paquier @ 2024-01-30 07:20 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jan 30, 2024 at 02:45:31PM +0900, Sutou Kouhei wrote:
> In <CAD21AoBmNiWwrspuedgAPgbAqsn7e7NoZYF6gNnYBf+gXEk9Mg@mail.gmail.com>
> "Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 11:11:59 +0900,
> Masahiko Sawada <[email protected]> wrote:
>
>> ---
>> + if (!format_specified)
>> + /* Set the default format. */
>> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
>> +
>>
>> I think we can pass "text" in this case instead of NULL. That way,
>> ProcessCopyOptionFormatTo doesn't need to handle NULL case.
>
> Yes, we can do it. But it needs a DefElem allocation. Is it
> acceptable?
I don't think that there is a need for a DelElem at all here? While I
am OK with the choice of calling CopyToInit() in the
ProcessCopyOption*() routines that exist to keep the set of callbacks
local to copyto.c and copyfrom.c, I think that this should not bother
about setting opts_out->csv_mode or opts_out->csv_mode but just set
the opts_out->{to,from}_routine callbacks.
>> +static void
>> +CopyToTextBasedInit(CopyToState cstate)
>> +{
>> +}
>>
>> and
>>
>> +static void
>> +CopyToBinaryInit(CopyToState cstate)
>> +{
>> +}
>>
>> Do we really need separate callbacks for the same behavior? I think we
>> can have a common init function say CopyToBuitinInit() that does
>> nothing. Or we can make the init callback optional.
Keeping empty options does not strike as a bad idea, because this
forces extension developers to think about this code path rather than
just ignore it. Now, all the Init() callbacks are empty for the
in-core callbacks, so I think that we should just remove it entirely
for now. Let's keep the core patch a maximum simple. It is always
possible to build on top of it depending on what people need. It's
been mentioned that JSON would want that, but this also proves that we
just don't care about that for all the in-core callbacks, as well. I
would choose a minimalistic design for now.
>> + /* Get info about the columns we need to process. */
>> + cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs *
>> sizeof(Fmgr\Info));
>> + foreach(cur, cstate->attnumlist)
>> + {
>> + int attnum = lfirst_int(cur);
>> + Oid out_func_oid;
>> + bool isvarlena;
>> + Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1);
>> +
>> + getTypeOutputInfo(attr->atttypid, &out_func_oid, &isvarlena);
>> + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
>> + }
>>
>> Is preparing the out functions an extension's responsibility? I
>> thought the core could prepare them based on the overall format
>> specified by extensions, as long as the overall format matches the
>> actual data format to send. What do you think?
>
> Hmm. I want to keep the preparation as an extension's
> responsibility. Because it's not needed for all formats. For
> example, Apache Arrow FORMAT doesn't need it. And JSON
> FORMAT doesn't need it too because it use
> composite_to_json().
I agree that it could be really useful for extensions to be able to
force that. We already know that for the in-core formats we've cared
about being able to enforce the way data is handled in input and
output.
> It makes sense. I couldn't find the documentation when I
> wrote it but I found it now...:
> https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-COPY
>
> Is there recommended comment style to refer a documentation?
> "See doc/src/sgml/protocol.sgml for the CopyOutResponse
> message details" is OK?
There are a couple of places in the C code where we refer to SGML docs
when it comes to specific details, so using a method like that here to
avoid a duplication with the docs sounds sensible for me.
I would be really tempted to put my hands on this patch to put into
shape a minimal set of changes because I'm caring quite a lot about
the performance gains reported with the removal of the "if" checks in
the per-row callbacks, and that's one goal of this thread quite
independent on the extensibility. Sutou-san, would you be OK with
that?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-30 08:15 ` Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-30 08:15 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 16:20:54 +0900,
Michael Paquier <[email protected]> wrote:
>>> + if (!format_specified)
>>> + /* Set the default format. */
>>> + ProcessCopyOptionFormatTo(pstate, opts_out, cstate, NULL);
>>> +
>>>
>>> I think we can pass "text" in this case instead of NULL. That way,
>>> ProcessCopyOptionFormatTo doesn't need to handle NULL case.
>>
>> Yes, we can do it. But it needs a DefElem allocation. Is it
>> acceptable?
>
> I don't think that there is a need for a DelElem at all here?
We use defel->location for an error message. (We don't need
to set location for the default "text" DefElem.)
> While I
> am OK with the choice of calling CopyToInit() in the
> ProcessCopyOption*() routines that exist to keep the set of callbacks
> local to copyto.c and copyfrom.c, I think that this should not bother
> about setting opts_out->csv_mode or opts_out->csv_mode but just set
> the opts_out->{to,from}_routine callbacks.
OK. I'll keep opts_out->{csv_mode,binary} in copy.c.
> Now, all the Init() callbacks are empty for the
> in-core callbacks, so I think that we should just remove it entirely
> for now. Let's keep the core patch a maximum simple. It is always
> possible to build on top of it depending on what people need. It's
> been mentioned that JSON would want that, but this also proves that we
> just don't care about that for all the in-core callbacks, as well. I
> would choose a minimalistic design for now.
OK. Let's remove Init() callbacks from the first patch set.
> I would be really tempted to put my hands on this patch to put into
> shape a minimal set of changes because I'm caring quite a lot about
> the performance gains reported with the removal of the "if" checks in
> the per-row callbacks, and that's one goal of this thread quite
> independent on the extensibility. Sutou-san, would you be OK with
> that?
Yes, sure.
(We want to focus on the performance gains in the first
patch set and then focus on extensibility again, right?)
For the purpose, I think that the v7 patch set is more
suitable than the v9 patch set. The v7 patch set doesn't
include Init() callbacks, custom options validation support
or extra Copy{In,Out}Response support. But the v7 patch set
misses the removal of the "if" checks in
NextCopyFromRawFields() that exists in the v9 patch set. I'm
not sure how much performance will improve by this but it
may be worth a try.
Can I prepare the v10 patch set as "the v7 patch set" + "the
removal of the "if" checks in NextCopyFromRawFields()"?
(+ reverting opts_out->{csv_mode,binary} changes in
ProcessCopyOptions().)
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-30 08:37 ` Michael Paquier <[email protected]>
2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Michael Paquier @ 2024-01-30 08:37 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Tue, Jan 30, 2024 at 05:15:11PM +0900, Sutou Kouhei wrote:
> We use defel->location for an error message. (We don't need
> to set location for the default "text" DefElem.)
Yeah, but you should not need to have this error in the paths that set
the callback routines in opts_out if the same validation happens a few
lines before, in copy.c.
> Yes, sure.
> (We want to focus on the performance gains in the first
> patch set and then focus on extensibility again, right?)
Yep, exactly, the numbers are too good to just ignore. I don't want
to hijack the thread, but I am really worried about the complexities
this thread is getting into because we are trying to shape the
callbacks in the most generic way possible based on *two* use cases.
This is going to be a never-ending discussion. I'd rather get some
simple basics, and then we can discuss if tweaking the callbacks is
really necessary or not. Even after introducing the pg_proc lookups
to get custom callbacks.
> For the purpose, I think that the v7 patch set is more
> suitable than the v9 patch set. The v7 patch set doesn't
> include Init() callbacks, custom options validation support
> or extra Copy{In,Out}Response support. But the v7 patch set
> misses the removal of the "if" checks in
> NextCopyFromRawFields() that exists in the v9 patch set. I'm
> not sure how much performance will improve by this but it
> may be worth a try.
Yeah.. The custom options don't seem like an absolute strong
requirement for the first shot with the callbacks or even the
possibility to retrieve the callbacks from a function call. I mean,
you could provide some control with SET commands and a few GUCs, at
least, even if that would be strange. Manipulations with a list of
DefElems is the intuitive way to have custom options at query level,
but we also have to guess the set of callbacks from this list of
DefElems coming from the query. You see my point, I am not sure
if it would be the best thing to process twice the options, especially
when it comes to decide if a DefElem should be valid or not depending
on the callbacks used. Or we could use a kind of "special" DefElem
where we could store a set of key:value fed to a callback :)
> Can I prepare the v10 patch set as "the v7 patch set" + "the
> removal of the "if" checks in NextCopyFromRawFields()"?
> (+ reverting opts_out->{csv_mode,binary} changes in
> ProcessCopyOptions().)
Yep, if I got it that would make sense to me. If you can do that,
that would help quite a bit. :)
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
@ 2024-01-31 05:11 ` Sutou Kouhei <[email protected]>
2024-01-31 05:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
0 siblings, 1 reply; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-31 05:11 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Tue, 30 Jan 2024 17:37:35 +0900,
Michael Paquier <[email protected]> wrote:
>> We use defel->location for an error message. (We don't need
>> to set location for the default "text" DefElem.)
>
> Yeah, but you should not need to have this error in the paths that set
> the callback routines in opts_out if the same validation happens a few
> lines before, in copy.c.
Ah, yes. defel->location is used in later patches. For
example, it's used when a COPY handler for the specified
FORMAT isn't found.
> I am really worried about the complexities
> this thread is getting into because we are trying to shape the
> callbacks in the most generic way possible based on *two* use cases.
> This is going to be a never-ending discussion. I'd rather get some
> simple basics, and then we can discuss if tweaking the callbacks is
> really necessary or not. Even after introducing the pg_proc lookups
> to get custom callbacks.
I understand your concern. Let's introduce minimal callbacks
as the first step. I think that we completed our design
discussion for this feature. We can choose minimal callbacks
based on the discussion.
> The custom options don't seem like an absolute strong
> requirement for the first shot with the callbacks or even the
> possibility to retrieve the callbacks from a function call. I mean,
> you could provide some control with SET commands and a few GUCs, at
> least, even if that would be strange. Manipulations with a list of
> DefElems is the intuitive way to have custom options at query level,
> but we also have to guess the set of callbacks from this list of
> DefElems coming from the query. You see my point, I am not sure
> if it would be the best thing to process twice the options, especially
> when it comes to decide if a DefElem should be valid or not depending
> on the callbacks used. Or we could use a kind of "special" DefElem
> where we could store a set of key:value fed to a callback :)
Interesting. Let's remove custom options support from the
initial minimal callbacks.
>> Can I prepare the v10 patch set as "the v7 patch set" + "the
>> removal of the "if" checks in NextCopyFromRawFields()"?
>> (+ reverting opts_out->{csv_mode,binary} changes in
>> ProcessCopyOptions().)
>
> Yep, if I got it that would make sense to me. If you can do that,
> that would help quite a bit. :)
I've prepared the v10 patch set. Could you try this?
Changes since the v7 patch set:
0001:
* Remove CopyToProcessOption() callback
* Remove CopyToGetFormat() callback
* Revert passing CopyToState to ProcessCopyOptions()
* Revert moving "opts_out->{csv_mode,binary} = true" to
ProcessCopyOptionFormatTo()
* Change to receive "const char *format" instead "DefElem *defel"
by ProcessCopyOptionFormatTo()
0002:
* Remove CopyFromProcessOption() callback
* Remove CopyFromGetFormat() callback
* Change to receive "const char *format" instead "DefElem
*defel" by ProcessCopyOptionFormatFrom()
* Remove "if (cstate->opts.csv_mode)" branches from
NextCopyFromRawFields()
FYI: Here are Copy{From,To}Routine in the v10 patch set. I
think that only Copy{From,To}OneRow are minimal callbacks
for the performance gain. But can we keep Copy{From,To}Start
and Copy{From,To}End for consistency? We can remove a few
{csv_mode,binary} conditions by Copy{From,To}{Start,End}. It
doesn't depend on the number of COPY target tuples. So they
will not affect performance.
/* Routines for a COPY FROM format implementation. */
typedef struct CopyFromRoutine
{
/*
* Called when COPY FROM is started. This will initialize something and
* receive a header.
*/
void (*CopyFromStart) (CopyFromState cstate, TupleDesc tupDesc);
/* Copy one row. It returns false if no more tuples. */
bool (*CopyFromOneRow) (CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls);
/* Called when COPY FROM is ended. This will finalize something. */
void (*CopyFromEnd) (CopyFromState cstate);
} CopyFromRoutine;
/* Routines for a COPY TO format implementation. */
typedef struct CopyToRoutine
{
/* Called when COPY TO is started. This will send a header. */
void (*CopyToStart) (CopyToState cstate, TupleDesc tupDesc);
/* Copy one row for COPY TO. */
void (*CopyToOneRow) (CopyToState cstate, TupleTableSlot *slot);
/* Called when COPY TO is ended. This will send a trailer. */
void (*CopyToEnd) (CopyToState cstate);
} CopyToRoutine;
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-31 05:39 ` Michael Paquier <[email protected]>
0 siblings, 0 replies; 51+ messages in thread
From: Michael Paquier @ 2024-01-31 05:39 UTC (permalink / raw)
To: Sutou Kouhei <[email protected]>; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
On Wed, Jan 31, 2024 at 02:11:22PM +0900, Sutou Kouhei wrote:
> Ah, yes. defel->location is used in later patches. For
> example, it's used when a COPY handler for the specified
> FORMAT isn't found.
I see.
> I've prepared the v10 patch set. Could you try this?
Thanks, I'm looking into that now.
> FYI: Here are Copy{From,To}Routine in the v10 patch set. I
> think that only Copy{From,To}OneRow are minimal callbacks
> for the performance gain. But can we keep Copy{From,To}Start
> and Copy{From,To}End for consistency? We can remove a few
> {csv_mode,binary} conditions by Copy{From,To}{Start,End}. It
> doesn't depend on the number of COPY target tuples. So they
> will not affect performance.
I think I'm OK to keep the start/end callbacks. This makes the code
more consistent as a whole, as well.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 51+ messages in thread
* Re: Make COPY format extendable: Extract COPY TO format implementations
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
@ 2024-01-24 14:20 ` Sutou Kouhei <[email protected]>
1 sibling, 0 replies; 51+ messages in thread
From: Sutou Kouhei @ 2024-01-24 14:20 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]; [email protected]; [email protected]; pgsql-hackers
Hi,
In <[email protected]>
"Re: Make COPY format extendable: Extract COPY TO format implementations" on Wed, 24 Jan 2024 14:49:36 +0900 (JST),
Sutou Kouhei <[email protected]> wrote:
> I've implemented custom COPY format feature based on the
> current design discussion. See the attached patches for
> details.
I forgot to mention one note. Documentation isn't included
in these patches. I'll write it after all (or some) patches
are merged. Is it OK?
Thanks,
--
kou
^ permalink raw reply [nested|flat] 51+ messages in thread
end of thread, other threads:[~2024-01-31 05:39 UTC | newest]
Thread overview: 51+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2023-01-23 19:47 [PATCH v10 4/4] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v18 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v14 7/7] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v20 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v16 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v20 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v13 7/7] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v18 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v19 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v12 6/6] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v19 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v19 5/5] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v15 7/7] introduce restore_library Nathan Bossart <[email protected]>
2023-02-16 06:06 [PATCH v18 5/5] introduce restore_library Nathan Bossart <[email protected]>
2024-01-15 06:27 Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 05:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-24 08:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 12:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Andrew Dunstan <[email protected]>
2024-01-24 14:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 02:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations jian he <[email protected]>
2024-01-25 03:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:05 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 03:17 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 08:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 23:35 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-26 08:49 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-25 04:36 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 04:53 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-25 05:28 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-25 08:52 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:18 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:32 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 08:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-26 08:55 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-26 09:02 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-27 06:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 06:03 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-29 06:48 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 02:41 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:10 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 03:21 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-29 03:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Junwang Zhao <[email protected]>
2024-01-29 09:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 02:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Masahiko Sawada <[email protected]>
2024-01-30 05:45 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 07:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-30 08:15 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-30 08:37 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-31 05:11 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
2024-01-31 05:39 ` Re: Make COPY format extendable: Extract COPY TO format implementations Michael Paquier <[email protected]>
2024-01-24 14:20 ` Re: Make COPY format extendable: Extract COPY TO format implementations Sutou Kouhei <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox