public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v7 2/2] Allow recovery via loadable modules. 9+ messages / 4 participants [nested] [flat]
* [PATCH v7 2/2] Allow recovery via loadable modules. @ 2022-12-10 03:40 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2022-12-10 03:40 UTC (permalink / raw) This adds the restore_library parameter to allow archive recovery via a loadable module, rather than running shell commands. --- contrib/basic_archive/Makefile | 4 +- contrib/basic_archive/basic_archive.c | 67 ++++++- contrib/basic_archive/meson.build | 7 +- contrib/basic_archive/t/001_restore.pl | 44 +++++ doc/src/sgml/archive-modules.sgml | 168 ++++++++++++++++-- doc/src/sgml/backup.sgml | 43 ++++- doc/src/sgml/basic-archive.sgml | 33 ++-- doc/src/sgml/config.sgml | 54 +++++- doc/src/sgml/high-availability.sgml | 23 ++- src/backend/access/transam/shell_restore.c | 21 ++- src/backend/access/transam/xlog.c | 13 +- src/backend/access/transam/xlogarchive.c | 70 +++++++- src/backend/access/transam/xlogrecovery.c | 26 ++- src/backend/postmaster/checkpointer.c | 26 +++ src/backend/postmaster/pgarch.c | 7 +- src/backend/postmaster/startup.c | 23 ++- src/backend/utils/misc/guc.c | 14 ++ src/backend/utils/misc/guc_tables.c | 10 ++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/access/xlog_internal.h | 1 + src/include/access/xlogarchive.h | 44 ++++- src/include/access/xlogrecovery.h | 1 + src/include/utils/guc.h | 2 + 23 files changed, 618 insertions(+), 84 deletions(-) create mode 100644 contrib/basic_archive/t/001_restore.pl diff --git a/contrib/basic_archive/Makefile b/contrib/basic_archive/Makefile index 55d299d650..487dc563f3 100644 --- a/contrib/basic_archive/Makefile +++ b/contrib/basic_archive/Makefile @@ -1,7 +1,7 @@ # contrib/basic_archive/Makefile MODULES = basic_archive -PGFILEDESC = "basic_archive - basic archive module" +PGFILEDESC = "basic_archive - basic archive and recovery module" REGRESS = basic_archive REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.conf @@ -9,6 +9,8 @@ REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/basic_archive/basic_archive.c # which typical installcheck users do not have (e.g. buildfarm clients). NO_INSTALLCHECK = 1 +TAP_TESTS = 1 + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/contrib/basic_archive/basic_archive.c b/contrib/basic_archive/basic_archive.c index 28cbb6cce0..8c333c8f99 100644 --- a/contrib/basic_archive/basic_archive.c +++ b/contrib/basic_archive/basic_archive.c @@ -17,6 +17,11 @@ * a file is successfully archived and then the system crashes before * a durable record of the success has been made. * + * This file also demonstrates a basic restore library implementation that + * is roughly equivalent to the following shell command: + * + * cp /path/to/archivedir/%f %p + * * Copyright (c) 2022-2023, PostgreSQL Global Development Group * * IDENTIFICATION @@ -30,6 +35,7 @@ #include <sys/time.h> #include <unistd.h> +#include "access/xlogarchive.h" #include "common/int.h" #include "miscadmin.h" #include "postmaster/pgarch.h" @@ -48,6 +54,8 @@ static bool basic_archive_file(const char *file, const char *path); static void basic_archive_file_internal(const char *file, const char *path); static bool check_archive_directory(char **newval, void **extra, GucSource source); static bool compare_files(const char *file1, const char *file2); +static bool basic_restore_file(const char *file, const char *path, + const char *lastRestartPointFileName); /* * _PG_init @@ -87,6 +95,19 @@ _PG_archive_module_init(ArchiveModuleCallbacks *cb) cb->archive_file_cb = basic_archive_file; } +/* + * _PG_recovery_module_init + * + * Returns the module's restore callback. + */ +void +_PG_recovery_module_init(RecoveryModuleCallbacks *cb) +{ + AssertVariableIsOfType(&_PG_recovery_module_init, RecoveryModuleInit); + + cb->restore_cb = basic_restore_file; +} + /* * check_archive_directory * @@ -99,8 +120,8 @@ check_archive_directory(char **newval, void **extra, GucSource source) /* * The default value is an empty string, so we have to accept that value. - * Our check_configured callback also checks for this and prevents - * archiving from proceeding if it is still empty. + * Our check_configured and restore callbacks also check for this and + * prevent archiving or recovery from proceeding if it is still empty. */ if (*newval == NULL || *newval[0] == '\0') return true; @@ -368,3 +389,45 @@ compare_files(const char *file1, const char *file2) return ret; } + +/* + * basic_restore_file + * + * Retrieves one file from the WAL archives. + */ +static bool +basic_restore_file(const char *file, const char *path, + const char *lastRestartPointFileName) +{ + char source[MAXPGPATH]; + struct stat st; + + ereport(DEBUG1, + (errmsg("restoring \"%s\" via basic_archive", file))); + + if (archive_directory == NULL || archive_directory[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"basic_archive.archive_directory\" is not set"))); + + /* + * Check whether the file exists. If not, we return false to indicate that + * there are no more files to restore. + */ + snprintf(source, MAXPGPATH, "%s/%s", archive_directory, file); + if (stat(source, &st) != 0) + { + int elevel = (errno == ENOENT) ? DEBUG1 : ERROR; + + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", source))); + return false; + } + + copy_file(source, unconstify(char *, path)); + + ereport(DEBUG1, + (errmsg("restored \"%s\" via basic_archive", file))); + return true; +} diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build index bc1380e6f6..af4580dea9 100644 --- a/contrib/basic_archive/meson.build +++ b/contrib/basic_archive/meson.build @@ -7,7 +7,7 @@ basic_archive_sources = files( if host_system == 'windows' basic_archive_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ '--NAME', 'basic_archive', - '--FILEDESC', 'basic_archive - basic archive module',]) + '--FILEDESC', 'basic_archive - basic archive and recovery module',]) endif basic_archive = shared_module('basic_archive', @@ -31,4 +31,9 @@ tests += { # which typical runningcheck users do not have (e.g. buildfarm clients). 'runningcheck': false, }, + 'tap': { + 'tests': [ + 't/001_restore.pl', + ], + }, } diff --git a/contrib/basic_archive/t/001_restore.pl b/contrib/basic_archive/t/001_restore.pl new file mode 100644 index 0000000000..ec8767d740 --- /dev/null +++ b/contrib/basic_archive/t/001_restore.pl @@ -0,0 +1,44 @@ + +# Copyright (c) 2022, PostgreSQL Global Development Group + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# start a node +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init(has_archiving => 1, allows_streaming => 1); +my $archive_dir = $node->archive_dir; +$archive_dir =~ s!\\!/!g if $PostgreSQL::Test::Utils::windows_os; +$node->append_conf('postgresql.conf', "archive_command = ''"); +$node->append_conf('postgresql.conf', "archive_library = 'basic_archive'"); +$node->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'"); +$node->start; + +# backup the node +my $backup = 'backup'; +$node->backup($backup); + +# generate some new WAL files +$node->safe_psql('postgres', "CREATE TABLE test (a INT);"); +$node->safe_psql('postgres', "SELECT pg_switch_wal();"); +$node->safe_psql('postgres', "INSERT INTO test VALUES (1);"); + +# shut down the node (this should archive all WAL files) +$node->stop; + +# restore from the backup +my $restore = PostgreSQL::Test::Cluster->new('restore'); +$restore->init_from_backup($node, $backup, has_restoring => 1, standby => 0); +$restore->append_conf('postgresql.conf', "restore_command = ''"); +$restore->append_conf('postgresql.conf', "restore_library = 'basic_archive'"); +$restore->append_conf('postgresql.conf', "basic_archive.archive_directory = '$archive_dir'"); +$restore->start; + +# ensure post-backup WAL was replayed +my $result = $restore->safe_psql("postgres", "SELECT count(*) FROM test;"); +is($result, "1", "check restore content"); + +done_testing(); diff --git a/doc/src/sgml/archive-modules.sgml b/doc/src/sgml/archive-modules.sgml index ef02051f7f..53e657040b 100644 --- a/doc/src/sgml/archive-modules.sgml +++ b/doc/src/sgml/archive-modules.sgml @@ -1,34 +1,40 @@ <!-- doc/src/sgml/archive-modules.sgml --> <chapter id="archive-modules"> - <title>Archive Modules</title> + <title>Archive and Recovery Modules</title> <indexterm zone="archive-modules"> - <primary>Archive Modules</primary> + <primary>Archive and Recovery Modules</primary> </indexterm> <para> PostgreSQL provides infrastructure to create custom modules for continuous - archiving (see <xref linkend="continuous-archiving"/>). While archiving via - a shell command (i.e., <xref linkend="guc-archive-command"/>) is much - simpler, a custom archive module will often be considerably more robust and - performant. + archiving and recovery (see <xref linkend="continuous-archiving"/>). While + a shell command (e.g., <xref linkend="guc-archive-command"/>, + <xref linkend="guc-restore-command"/>) is much simpler, a custom module will + often be considerably more robust and performant. </para> <para> When a custom <xref linkend="guc-archive-library"/> is configured, PostgreSQL will submit completed WAL files to the module, and the server will avoid recycling or removing these WAL files until the module indicates that the files - were successfully archived. It is ultimately up to the module to decide what - to do with each WAL file, but many recommendations are listed at - <xref linkend="backup-archiving-wal"/>. + were successfully archived. When a custom + <xref linkend="guc-restore-library"/> is configured, PostgreSQL will use the + module for recovery actions. It is ultimately up to the module to decide how + to accomplish each task, but some recommendations are listed at + <xref linkend="backup-archiving-wal"/> and + <xref linkend="backup-pitr-recovery"/>. </para> <para> - Archiving modules must at least consist of an initialization function (see - <xref linkend="archive-module-init"/>) and the required callbacks (see - <xref linkend="archive-module-callbacks"/>). However, archive modules are - also permitted to do much more (e.g., declare GUCs and register background - workers). + Archive and recovery modules must at least consist of an initialization + function (see <xref linkend="archive-module-init"/> and + <xref linkend="recovery-module-init"/>) and the required callbacks (see + <xref linkend="archive-module-callbacks"/> and + <xref linkend="recovery-module-callbacks"/>). However, archive and recovery + modules are also permitted to do much more (e.g., declare GUCs and register + background workers). A module may be used for both + <varname>archive_library</varname> and <varname>restore_library</varname>. </para> <para> @@ -37,7 +43,7 @@ </para> <sect1 id="archive-module-init"> - <title>Initialization Functions</title> + <title>Archive Module Initialization Functions</title> <indexterm zone="archive-module-init"> <primary>_PG_archive_module_init</primary> </indexterm> @@ -64,6 +70,12 @@ typedef void (*ArchiveModuleInit) (struct ArchiveModuleCallbacks *cb); Only the <function>archive_file_cb</function> callback is required. The others are optional. </para> + + <note> + <para> + <varname>archive_library</varname> is only loaded in the archiver process. + </para> + </note> </sect1> <sect1 id="archive-module-callbacks"> @@ -129,6 +141,132 @@ typedef bool (*ArchiveFileCB) (const char *file, const char *path); <programlisting> typedef void (*ArchiveShutdownCB) (void); +</programlisting> + </para> + </sect2> + </sect1> + + <sect1 id="recovery-module-init"> + <title>Recovery Module Initialization Functions</title> + <indexterm zone="recovery-module-init"> + <primary>_PG_recovery_module_init</primary> + </indexterm> + <para> + A recovery library is loaded by dynamically loading a shared library with the + <xref linkend="guc-restore-library"/> as the library base name. The normal + library search path is used to locate the library. To provide the required + recovery module callbacks and to indicate that the library is actually a + recovery module, it needs to provide a function named + <function>_PG_recovery_module_init</function>. This function is passed a + struct that needs to be filled with the callback function pointers for + individual actions. + +<programlisting> +typedef struct RecoveryModuleCallbacks +{ + RecoveryRestoreCB restore_cb; + RecoveryArchiveCleanupCB archive_cleanup_cb; + RecoveryEndCB recovery_end_cb; + RecoveryShutdownCB shutdown_cb; +} RecoveryModuleCallbacks; +typedef void (*RecoveryModuleInit) (struct RecoveryModuleCallbacks *cb); +</programlisting> + + The <function>restore_cb</function> callback is required for archive + recovery, but it is optional for streaming replication. The others are + always optional. + </para> + + <note> + <para> + <varname>restore_library</varname> is only loaded in the startup and + checkpointer processes and in single-user mode. + </para> + </note> + </sect1> + + <sect1 id="recovery-module-callbacks"> + <title>Recovery Module Callbacks</title> + <para> + The recovery callbacks define the actual behavior of the module. The server + will call them as required to execute recovery actions. + </para> + + <sect2 id="recovery-module-restore"> + <title>Restore Callback</title> + <para> + The <function>restore_cb</function> callback is called to retrieve a single + archived segment of the WAL file series for archive recovery or streaming + replication. + +<programlisting> +typedef bool (*RecoveryRestoreCB) (const char *file, const char *path, const char *lastRestartPointFileName); +</programlisting> + + This callback must return <literal>true</literal> only if the file was + successfully retrieved. If the file is not available in the archives, the + callback must return <literal>false</literal>. + <replaceable>file</replaceable> will contain just the file name + of the WAL file to retrieve, while <replaceable>path</replaceable> contains + the destination's relative path (including the file name). + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point. That is the earliest + file that must be kept to allow a restore to be restartable, so this + information can be used to truncate the archive to just the minimum + required to support restarting from the current restore. + <replaceable>lastRestartPointFileName</replaceable> is typically only used + by warm-standby configurations (see <xref linkend="warm-standby"/>). Note + that if multiple standby servers are restoring from the same archive + directory, you will need to ensure that you do not delete WAL files until + they are no longer needed by any of the servers. + </para> + </sect2> + + <sect2 id="recovery-module-archive-cleanup"> + <title>Archive Cleanup Callback</title> + <para> + The <function>archive_cleanup_cb</function> callback is called at every + restart point and is intended to provide a mechanism for cleaning up old + archived WAL files that are no longer needed by the standby server. + +<programlisting> +typedef void (*RecoveryArchiveCleanupCB) (const char *lastRestartPointFileName); +</programlisting> + + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point, like in + <link linkend="recovery-module-restore"><function>restore_cb</function></link>. + </para> + </sect2> + + <sect2 id="recovery-module-end"> + <title>Recovery End Callback</title> + <para> + The <function>recovery_end_cb</function> callback is called once at the end + of recovery and is intended to provide a mechanism for cleanup following + replication or recovery. + +<programlisting> +typedef void (*RecoveryEndCB) (const char *lastRestartPointFileName); +</programlisting> + + <replaceable>lastRestartPointFileName</replaceable> will contain the name + of the file containing the last valid restart point, like in + <link linkend="recovery-module-restore"><function>restore_cb</function></link>. + </para> + </sect2> + + <sect2 id="recovery-module-shutdown"> + <title>Shutdown Callback</title> + <para> + The <function>shutdown_cb</function> callback is called when a process that + has loaded the recovery module exits (e.g., after an error) or the value of + <xref linkend="guc-restore-library"/> changes. If no + <function>shutdown_cb</function> is defined, no special action is taken in + these situations. + +<programlisting> +typedef void (*RecoveryShutdownCB) (void); </programlisting> </para> </sect2> diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index be05a33205..f44135061d 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1180,9 +1180,27 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true); <para> The key part of all this is to set up a recovery configuration that describes how you want to recover and how far the recovery should - run. The one thing that you absolutely must specify is the <varname>restore_command</varname>, - which tells <productname>PostgreSQL</productname> how to retrieve archived - WAL file segments. Like the <varname>archive_command</varname>, this is + run. The one thing that you absolutely must specify is either + <varname>restore_command</varname> or a <varname>restore_library</varname> + that defines a restore callback, which tells + <productname>PostgreSQL</productname> how to retrieve archived WAL file + segments. + </para> + + <para> + Like the <varname>archive_library</varname> parameter, + <varname>restore_library</varname> is a shared library. Since such + libraries are written in <literal>C</literal>, creating your own may + require considerably more effort than writing a shell command. However, + recovery modules can be more performant than restoring via shell, and they + will have access to many useful server resources. For more information + about creating a <varname>restore_library</varname>, see + <xref linkend="archive-modules"/>. + </para> + + <para> + Like the <varname>archive_command</varname>, + <varname>restore_command</varname> is a shell command string. It can contain <literal>%f</literal>, which is replaced by the name of the desired WAL file, and <literal>%p</literal>, which is replaced by the path name to copy the WAL file to. @@ -1201,14 +1219,20 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' </para> <para> - It is important that the command return nonzero exit status on failure. - The command <emphasis>will</emphasis> be called requesting files that are not - present in the archive; it must return nonzero when so asked. This is not - an error condition. An exception is that if the command was terminated by + It is important that the <varname>restore_command</varname> return nonzero + exit status on failure, or, if you are using a + <varname>restore_library</varname>, that the restore function returns + <literal>false</literal> on failure. The command or library + <emphasis>will</emphasis> be called requesting files that are not + present in the archive; it must fail when so asked. This is not + an error condition. An exception is that if the + <varname>restore_command</varname> was terminated by a signal (other than <systemitem>SIGTERM</systemitem>, which is used as part of a database server shutdown) or an error by the shell (such as command not found), then recovery will abort and the server will not start - up. + up. Likewise, if the restore function provided by the + <varname>restore_library</varname> emits an <literal>ERROR</literal> or + <literal>FATAL</literal>, recovery will abort and the server won't start. </para> <para> @@ -1232,7 +1256,8 @@ restore_command = 'cp /mnt/server/archivedir/%f %p' close as possible given the available WAL segments). Therefore, a normal recovery will end with a <quote>file not found</quote> message, the exact text of the error message depending upon your choice of - <varname>restore_command</varname>. You may also see an error message + <varname>restore_command</varname> or <varname>restore_library</varname>. + You may also see an error message at the start of recovery for a file named something like <filename>00000001.history</filename>. This is also normal and does not indicate a problem in simple recovery situations; see diff --git a/doc/src/sgml/basic-archive.sgml b/doc/src/sgml/basic-archive.sgml index 60f23d2855..11fd670dbc 100644 --- a/doc/src/sgml/basic-archive.sgml +++ b/doc/src/sgml/basic-archive.sgml @@ -8,17 +8,20 @@ </indexterm> <para> - <filename>basic_archive</filename> is an example of an archive module. This - module copies completed WAL segment files to the specified directory. This - may not be especially useful, but it can serve as a starting point for - developing your own archive module. For more information about archive - modules, see <xref linkend="archive-modules"/>. + <filename>basic_archive</filename> is an example of an archive and recovery + module. This module copies completed WAL segment files to or from the + specified directory. This may not be especially useful, but it can serve as + a starting point for developing your own archive and recovery modules. For + more information about archive and recovery modules, see + see <xref linkend="archive-modules"/>. </para> <para> - In order to function, this module must be loaded via + For use as an archive module, this module must be loaded via <xref linkend="guc-archive-library"/>, and <xref linkend="guc-archive-mode"/> - must be enabled. + must be enabled. For use as a recovery module, this module must be loaded + via <xref linkend="guc-restore-library"/>, and recovery must be enabled (see + <xref linkend="runtime-config-wal-archive-recovery"/>). </para> <sect2 id="basic-archive-configuration-parameters"> @@ -34,11 +37,12 @@ </term> <listitem> <para> - The directory where the server should copy WAL segment files. This - directory must already exist. The default is an empty string, which - effectively halts WAL archiving, but if <xref linkend="guc-archive-mode"/> - is enabled, the server will accumulate WAL segment files in the - expectation that a value will soon be provided. + The directory where the server should copy WAL segment files to or from. + This directory must already exist. The default is an empty string, + which, when used for archiving, effectively halts WAL archival, but if + <xref linkend="guc-archive-mode"/> is enabled, the server will accumulate + WAL segment files in the expectation that a value will soon be provided. + When an empty string is used for recovery, restore will fail. </para> </listitem> </varlistentry> @@ -46,7 +50,7 @@ <para> These parameters must be set in <filename>postgresql.conf</filename>. - Typical usage might be: + Typical usage as an archive module might be: </para> <programlisting> @@ -61,7 +65,8 @@ basic_archive.archive_directory = '/path/to/archive/directory' <title>Notes</title> <para> - Server crashes may leave temporary files with the prefix + When <filename>basic_archive</filename> is used as an archive module, server + crashes may leave temporary files with the prefix <filename>archtemp</filename> in the archive directory. It is recommended to delete such files before restarting the server after a crash. It is safe to remove such files while the server is running as long as they are unrelated diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 77574e2d4e..039d3360ac 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3773,7 +3773,8 @@ include_dir 'conf.d' recovery when the end of archived WAL is reached, but will keep trying to continue recovery by connecting to the sending server as specified by the <varname>primary_conninfo</varname> setting and/or by fetching new WAL - segments using <varname>restore_command</varname>. For this mode, the + segments using <varname>restore_command</varname> or + <varname>restore_library</varname>. For this mode, the parameters from this section and <xref linkend="runtime-config-replication-standby"/> are of interest. Parameters from <xref linkend="runtime-config-wal-recovery-target"/> will @@ -3801,7 +3802,8 @@ include_dir 'conf.d' <listitem> <para> The local shell command to execute to retrieve an archived segment of - the WAL file series. This parameter is required for archive recovery, + the WAL file series. Either <varname>restore_command</varname> or + <xref linkend="guc-restore-library"/> is required for archive recovery, but optional for streaming replication. Any <literal>%f</literal> in the string is replaced by the name of the file to retrieve from the archive, @@ -3836,7 +3838,42 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>restore_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. + </para> + </listitem> + </varlistentry> + + <varlistentry id="guc-restore-library" xreflabel="restore_library"> + <term><varname>restore_library</varname> (<type>string</type>) + <indexterm> + <primary><varname>restore_library</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + The library to use for recovery actions, including retrieving archived + segments of the WAL file series and executing tasks at restartpoints + and at recovery end. Either <xref linkend="guc-restore-command"/> or + <varname>restore_library</varname> is required for archive recovery, + but optional for streaming replication. If this parameter is set to an + empty string (the default), restoring via shell is enabled, and + <varname>restore_command</varname>, + <varname>archive_cleanup_command</varname> and + <varname>recovery_end_command</varname> are used. If both + <varname>restore_library</varname> and any of + <varname>restore_command</varname>, + <varname>archive_cleanup_command</varname> or + <varname>recovery_end_command</varname> are set, an error will be + raised. Otherwise, the specified shared library is used for recovery. + For more information, see <xref linkend="archive-modules"/>. + </para> + + <para> + This parameter can only be set in the + <filename>postgresql.conf</filename> file or on the server command line. </para> </listitem> </varlistentry> @@ -3881,7 +3918,10 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>archive_cleanup_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. </para> </listitem> </varlistentry> @@ -3910,11 +3950,13 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows </para> <para> This parameter can only be set in the <filename>postgresql.conf</filename> - file or on the server command line. + file or on the server command line. It is only used if + <varname>restore_library</varname> is set to an empty string. If both + <varname>recovery_end_command</varname> and + <varname>restore_library</varname> are set, an error will be raised. </para> </listitem> </varlistentry> - </variablelist> </sect2> diff --git a/doc/src/sgml/high-availability.sgml b/doc/src/sgml/high-availability.sgml index f180607528..6266e2df7f 100644 --- a/doc/src/sgml/high-availability.sgml +++ b/doc/src/sgml/high-availability.sgml @@ -627,7 +627,8 @@ protocol to make nodes agree on a serializable transactional order. <para> In standby mode, the server continuously applies WAL received from the primary server. The standby server can read WAL from a WAL archive - (see <xref linkend="guc-restore-command"/>) or directly from the primary + (see <xref linkend="guc-restore-command"/> and + <xref linkend="guc-restore-library"/>) or directly from the primary over a TCP connection (streaming replication). The standby server will also attempt to restore any WAL found in the standby cluster's <filename>pg_wal</filename> directory. That typically happens after a server @@ -638,9 +639,11 @@ protocol to make nodes agree on a serializable transactional order. <para> At startup, the standby begins by restoring all WAL available in the - archive location, calling <varname>restore_command</varname>. Once it - reaches the end of WAL available there and <varname>restore_command</varname> - fails, it tries to restore any WAL available in the <filename>pg_wal</filename> directory. + archive location, either by calling <varname>restore_command</varname> or + by executing the <varname>restore_library</varname>'s restore callback. + Once it reaches the end of WAL available there and + <varname>restore_command</varname> or the restore callback fails, it tries + to restore any WAL available in the <filename>pg_wal</filename> directory. If that fails, and streaming replication has been configured, the standby tries to connect to the primary server and start streaming WAL from the last valid record found in archive or <filename>pg_wal</filename>. If that fails @@ -698,7 +701,8 @@ protocol to make nodes agree on a serializable transactional order. server (see <xref linkend="backup-pitr-recovery"/>). Create a file <link linkend="file-standby-signal"><filename>standby.signal</filename></link><indexterm><primary>standby.signal</primary></indexterm> in the standby's cluster data - directory. Set <xref linkend="guc-restore-command"/> to a simple command to copy files from + directory. Set <xref linkend="guc-restore-command"/> or + <xref linkend="guc-restore-library"/> to copy files from the WAL archive. If you plan to have multiple standby servers for high availability purposes, make sure that <varname>recovery_target_timeline</varname> is set to <literal>latest</literal> (the default), to make the standby server follow the timeline change @@ -707,7 +711,8 @@ protocol to make nodes agree on a serializable transactional order. <note> <para> - <xref linkend="guc-restore-command"/> should return immediately + <xref linkend="guc-restore-command"/> and restore callbacks provided by + <xref linkend="guc-restore-library"/> should return immediately if the file does not exist; the server will retry the command again if necessary. </para> @@ -731,8 +736,10 @@ protocol to make nodes agree on a serializable transactional order. <para> If you're using a WAL archive, its size can be minimized using the <xref - linkend="guc-archive-cleanup-command"/> parameter to remove files that are no - longer required by the standby server. + linkend="guc-archive-cleanup-command"/> parameter or the + <xref linkend="guc-restore-library"/>'s + <function>archive_cleanup_cb</function> callback function to remove files + that are no longer required by the standby server. The <application>pg_archivecleanup</application> utility is designed specifically to be used with <varname>archive_cleanup_command</varname> in typical single-standby configurations, see <xref linkend="pgarchivecleanup"/>. diff --git a/src/backend/access/transam/shell_restore.c b/src/backend/access/transam/shell_restore.c index f5b6cf174e..c0bc78d8b4 100644 --- a/src/backend/access/transam/shell_restore.c +++ b/src/backend/access/transam/shell_restore.c @@ -4,7 +4,8 @@ * Recovery functions for a user-specified shell command. * * These recovery functions use a user-specified shell command (e.g. based - * on the GUC restore_command). + * on the GUC restore_command). It is used as the default, but other + * modules may define their own recovery logic. * * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -25,11 +26,25 @@ #include "storage/ipc.h" #include "utils/wait_event.h" +static bool shell_restore(const char *file, const char *path, + const char *lastRestartPointFileName); +static void shell_archive_cleanup(const char *lastRestartPointFileName); +static void shell_recovery_end(const char *lastRestartPointFileName); static bool ExecuteRecoveryCommand(const char *command, const char *commandName, bool failOnSignal, bool exitOnSigterm, uint32 wait_event_info, int fail_elevel); +void +shell_restore_init(RecoveryModuleCallbacks *cb) +{ + AssertVariableIsOfType(&shell_restore_init, RecoveryModuleInit); + + cb->restore_cb = shell_restore; + cb->archive_cleanup_cb = shell_archive_cleanup; + cb->recovery_end_cb = shell_recovery_end; +} + /* * Attempt to execute a shell-based restore command. * @@ -80,7 +95,7 @@ shell_restore(const char *file, const char *path, /* * Attempt to execute a shell-based archive cleanup command. */ -void +static void shell_archive_cleanup(const char *lastRestartPointFileName) { char *cmd; @@ -96,7 +111,7 @@ shell_archive_cleanup(const char *lastRestartPointFileName) /* * Attempt to execute a shell-based end-of-recovery command. */ -void +static void shell_recovery_end(const char *lastRestartPointFileName) { char *cmd; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8f47fb7570..ae537cd87f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4884,15 +4884,16 @@ static void CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog, TimeLineID newTLI) { + /* - * Execute the recovery_end_command, if any. + * Execute the recovery-end callback, if any. */ - if (recoveryEndCommand && strcmp(recoveryEndCommand, "") != 0) + if (RecoveryContext.recovery_end_cb) { char lastRestartPointFname[MAXFNAMELEN]; GetOldestRestartPointFileName(lastRestartPointFname); - shell_recovery_end(lastRestartPointFname); + RecoveryContext.recovery_end_cb(lastRestartPointFname); } /* @@ -7307,14 +7308,14 @@ CreateRestartPoint(int flags) timestamptz_to_str(xtime)) : 0)); /* - * Finally, execute archive_cleanup_command, if any. + * Execute the archive-cleanup callback, if any. */ - if (archiveCleanupCommand && strcmp(archiveCleanupCommand, "") != 0) + if (RecoveryContext.archive_cleanup_cb) { char lastRestartPointFname[MAXFNAMELEN]; GetOldestRestartPointFileName(lastRestartPointFname); - shell_archive_cleanup(lastRestartPointFname); + RecoveryContext.archive_cleanup_cb(lastRestartPointFname); } return true; diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index b5cb060d55..fdab7dad43 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -22,7 +22,9 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xlogarchive.h" +#include "access/xlogrecovery.h" #include "common/archive.h" +#include "fmgr.h" #include "miscadmin.h" #include "pgstat.h" #include "postmaster/startup.h" @@ -32,6 +34,11 @@ #include "storage/ipc.h" #include "storage/lwlock.h" +/* + * Global context for recovery-related callbacks. + */ +RecoveryModuleCallbacks RecoveryContext; + /* * Attempt to retrieve the specified file from off-line archival storage. * If successful, fill "path" with its complete path (note that this will be @@ -71,7 +78,7 @@ RestoreArchivedFile(char *path, const char *xlogfname, goto not_available; /* In standby mode, restore_command might not be supplied */ - if (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0) + if (RecoveryContext.restore_cb == NULL) goto not_available; /* @@ -149,14 +156,15 @@ RestoreArchivedFile(char *path, const char *xlogfname, XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size); /* - * Check signals before restore command and reset afterwards. + * Check signals before restore callback and reset afterwards. */ PreRestoreCommand(); /* * Copy xlog from archival storage to XLOGDIR */ - ret = shell_restore(xlogfname, xlogpath, lastRestartPointFname); + ret = RecoveryContext.restore_cb(xlogfname, xlogpath, + lastRestartPointFname); PostRestoreCommand(); @@ -603,3 +611,59 @@ XLogArchiveCleanup(const char *xlog) unlink(archiveStatusPath); /* should we complain about failure? */ } + +/* + * Loads all the recovery callbacks into our global RecoveryContext. The + * caller is responsible for validating the combination of library/command + * parameters that are set (e.g., restore_command and restore_library cannot + * both be set). + */ +void +LoadRecoveryCallbacks(void) +{ + RecoveryModuleInit init; + + /* + * If the shell command is enabled, use our special initialization + * function. Otherwise, load the library and call its + * _PG_recovery_module_init(). + */ + if (restoreLibrary[0] == '\0') + init = shell_restore_init; + else + init = (RecoveryModuleInit) + load_external_function(restoreLibrary, "_PG_recovery_module_init", + false, NULL); + + if (init == NULL) + ereport(ERROR, + (errmsg("recovery modules have to define the symbol " + "_PG_recovery_module_init"))); + + memset(&RecoveryContext, 0, sizeof(RecoveryModuleCallbacks)); + (*init) (&RecoveryContext); + + /* + * If using shell commands, remove callbacks for any commands that are not + * set. + */ + if (restoreLibrary[0] == '\0') + { + if (recoveryRestoreCommand[0] == '\0') + RecoveryContext.restore_cb = NULL; + if (archiveCleanupCommand[0] == '\0') + RecoveryContext.archive_cleanup_cb = NULL; + if (recoveryEndCommand[0] == '\0') + RecoveryContext.recovery_end_cb = NULL; + } +} + +/* + * Call the shutdown callback of the loaded recovery module, if defined. + */ +void +call_recovery_module_shutdown_cb(int code, Datum arg) +{ + if (RecoveryContext.shutdown_cb) + RecoveryContext.shutdown_cb(); +} diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 5e65785306..db0cd4469a 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -80,6 +80,7 @@ const struct config_enum_entry recovery_target_action_options[] = { /* options formerly taken from recovery.conf for archive recovery */ char *recoveryRestoreCommand = NULL; +char *restoreLibrary = NULL; char *recoveryEndCommand = NULL; char *archiveCleanupCommand = NULL; RecoveryTargetType recoveryTarget = RECOVERY_TARGET_UNSET; @@ -1053,24 +1054,37 @@ validateRecoveryParameters(void) if (!ArchiveRecoveryRequested) return; + /* + * Check for invalid combinations of the command/library parameters and + * load the callbacks. + */ + CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library", + recoveryRestoreCommand, "restore_command"); + CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library", + recoveryEndCommand, "recovery_end_command"); + before_shmem_exit(call_recovery_module_shutdown_cb, 0); + LoadRecoveryCallbacks(); + /* * Check for compulsory parameters */ if (StandbyModeRequested) { if ((PrimaryConnInfo == NULL || strcmp(PrimaryConnInfo, "") == 0) && - (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0)) + RecoveryContext.restore_cb == NULL) ereport(WARNING, - (errmsg("specified neither primary_conninfo nor restore_command"), - errhint("The database server will regularly poll the pg_wal subdirectory to check for files placed there."))); + (errmsg("specified neither primary_conninfo nor restore_command " + "nor a restore_library that defines a restore callback"), + errhint("The database server will regularly poll the pg_wal " + "subdirectory to check for files placed there."))); } else { - if (recoveryRestoreCommand == NULL || - strcmp(recoveryRestoreCommand, "") == 0) + if (RecoveryContext.restore_cb == NULL) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("must specify restore_command when standby mode is not enabled"))); + errmsg("must specify restore_command or a restore_library that defines " + "a restore callback when standby mode is not enabled"))); } /* diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index de0bbbfa79..6350fd0b83 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -38,6 +38,7 @@ #include "access/xlog.h" #include "access/xlog_internal.h" +#include "access/xlogarchive.h" #include "access/xlogrecovery.h" #include "libpq/pqsignal.h" #include "miscadmin.h" @@ -222,6 +223,16 @@ CheckpointerMain(void) */ before_shmem_exit(pgstat_before_server_shutdown, 0); + /* + * Check for invalid combinations of the command/library parameters and + * load the callbacks. We do this before setting up the exception handler + * so that any problems result in a server crash shortly after startup. + */ + CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library", + archiveCleanupCommand, "archive_cleanup_command"); + before_shmem_exit(call_recovery_module_shutdown_cb, 0); + LoadRecoveryCallbacks(); + /* * Create a memory context that we will do all our work in. We do this so * that we can reset the context during error recovery and thereby avoid @@ -548,6 +559,9 @@ HandleCheckpointerInterrupts(void) if (ConfigReloadPending) { + char *prevRestoreLibrary = pstrdup(restoreLibrary); + char *prevArchiveCleanupCommand = pstrdup(archiveCleanupCommand); + ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); @@ -563,6 +577,18 @@ HandleCheckpointerInterrupts(void) * because of SIGHUP. */ UpdateSharedMemoryConfig(); + + CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library", + archiveCleanupCommand, "archive_cleanup_command"); + if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 || + strcmp(prevArchiveCleanupCommand, archiveCleanupCommand) != 0) + { + call_recovery_module_shutdown_cb(0, (Datum) 0); + LoadRecoveryCallbacks(); + } + + pfree(prevRestoreLibrary); + pfree(prevArchiveCleanupCommand); } if (ShutdownRequestPending) { diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgarch.c index 8ecdb9ca23..8e91f2d70f 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -831,11 +831,8 @@ LoadArchiveLibrary(void) { ArchiveModuleInit archive_init; - if (XLogArchiveLibrary[0] != '\0' && XLogArchiveCommand[0] != '\0') - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("both archive_command and archive_library set"), - errdetail("Only one of archive_command, archive_library may be set."))); + CheckMutuallyExclusiveGUCs(XLogArchiveLibrary, "archive_library", + XLogArchiveCommand, "archive_command"); memset(&ArchiveContext, 0, sizeof(ArchiveModuleCallbacks)); diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index 8786186898..f9ff2b5583 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -20,6 +20,7 @@ #include "postgres.h" #include "access/xlog.h" +#include "access/xlogarchive.h" #include "access/xlogrecovery.h" #include "access/xlogutils.h" #include "libpq/pqsignal.h" @@ -133,13 +134,17 @@ StartupProcShutdownHandler(SIGNAL_ARGS) * Re-read the config file. * * If one of the critical walreceiver options has changed, flag xlog.c - * to restart it. + * to restart it. Also, check for invalid combinations of the command/library + * parameters and reload the recovery callbacks if necessary. */ static void StartupRereadConfig(void) { char *conninfo = pstrdup(PrimaryConnInfo); char *slotname = pstrdup(PrimarySlotName); + char *prevRestoreLibrary = pstrdup(restoreLibrary); + char *prevRestoreCommand = pstrdup(recoveryRestoreCommand); + char *prevRecoveryEndCommand = pstrdup(recoveryEndCommand); bool tempSlot = wal_receiver_create_temp_slot; bool conninfoChanged; bool slotnameChanged; @@ -161,6 +166,22 @@ StartupRereadConfig(void) if (conninfoChanged || slotnameChanged || tempSlotChanged) StartupRequestWalReceiverRestart(); + + CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library", + recoveryRestoreCommand, "restore_command"); + CheckMutuallyExclusiveGUCs(restoreLibrary, "restore_library", + recoveryEndCommand, "recovery_end_command"); + if (strcmp(prevRestoreLibrary, restoreLibrary) != 0 || + strcmp(prevRestoreCommand, recoveryRestoreCommand) != 0 || + strcmp(prevRecoveryEndCommand, recoveryEndCommand) != 0) + { + call_recovery_module_shutdown_cb(0, (Datum) 0); + LoadRecoveryCallbacks(); + } + + pfree(prevRestoreLibrary); + pfree(prevRestoreCommand); + pfree(prevRecoveryEndCommand); } /* Handle various signals that might be sent to the startup process */ diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index d52069f446..7858e9a649 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -6880,3 +6880,17 @@ call_enum_check_hook(struct config_enum *conf, int *newval, void **extra, return true; } + +/* + * ERROR if both parameters are set. + */ +void +CheckMutuallyExclusiveGUCs(const char *p1val, const char *p1name, + const char *p2val, const char *p2name) +{ + if (p1val[0] != '\0' && p2val[0] != '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("both %s and %s set", p1name, p2name), + errdetail("Only one of %s, %s may be set.", p1name, p2name))); +} diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5025e80f89..a8a516b0c6 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3776,6 +3776,16 @@ struct config_string ConfigureNamesString[] = NULL, NULL, NULL }, + { + {"restore_library", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY, + gettext_noop("Sets the library that will be called for recovery actions."), + NULL + }, + &restoreLibrary, + "", + NULL, NULL, NULL + }, + { {"archive_cleanup_command", PGC_SIGHUP, WAL_ARCHIVE_RECOVERY, gettext_noop("Sets the shell command that will be executed at every restart point."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 4cceda4162..38fb3e0823 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -269,6 +269,7 @@ # placeholders: %p = path of file to restore # %f = file name only # e.g. 'cp /mnt/server/archivedir/%f %p' +#restore_library = '' # library to use for recovery actions #archive_cleanup_command = '' # command to execute at every restartpoint #recovery_end_command = '' # command to execute at completion of recovery diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 59fc7bc105..756f0898b5 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -400,5 +400,6 @@ extern PGDLLIMPORT bool ArchiveRecoveryRequested; extern PGDLLIMPORT bool InArchiveRecovery; extern PGDLLIMPORT bool StandbyMode; extern PGDLLIMPORT char *recoveryRestoreCommand; +extern PGDLLIMPORT char *restoreLibrary; #endif /* XLOG_INTERNAL_H */ diff --git a/src/include/access/xlogarchive.h b/src/include/access/xlogarchive.h index 299304703e..71c9b88165 100644 --- a/src/include/access/xlogarchive.h +++ b/src/include/access/xlogarchive.h @@ -30,9 +30,45 @@ extern bool XLogArchiveIsReady(const char *xlog); extern bool XLogArchiveIsReadyOrDone(const char *xlog); extern void XLogArchiveCleanup(const char *xlog); -extern bool shell_restore(const char *file, const char *path, - const char *lastRestartPointFileName); -extern void shell_archive_cleanup(const char *lastRestartPointFileName); -extern void shell_recovery_end(const char *lastRestartPointFileName); +/* + * Recovery module callbacks + * + * These callback functions should be defined by recovery libraries and + * returned via _PG_recovery_module_init(). For more information about the + * purpose of each callback, refer to the recovery modules documentation. + */ +typedef bool (*RecoveryRestoreCB) (const char *file, const char *path, + const char *lastRestartPointFileName); +typedef void (*RecoveryArchiveCleanupCB) (const char *lastRestartPointFileName); +typedef void (*RecoveryEndCB) (const char *lastRestartPointFileName); +typedef void (*RecoveryShutdownCB) (void); + +typedef struct RecoveryModuleCallbacks +{ + RecoveryRestoreCB restore_cb; + RecoveryArchiveCleanupCB archive_cleanup_cb; + RecoveryEndCB recovery_end_cb; + RecoveryShutdownCB shutdown_cb; +} RecoveryModuleCallbacks; + +extern RecoveryModuleCallbacks RecoveryContext; + +/* + * Type of the shared library symbol _PG_recovery_module_init that is looked up + * when loading a recovery library. + */ +typedef void (*RecoveryModuleInit) (RecoveryModuleCallbacks *cb); + +extern PGDLLEXPORT void _PG_recovery_module_init(RecoveryModuleCallbacks *cb); + +extern void LoadRecoveryCallbacks(void); +extern void call_recovery_module_shutdown_cb(int code, Datum arg); + +/* + * Since the logic for recovery via a shell command is in the core server and + * does not need to be loaded via a shared library, it has a special + * initialization function. + */ +extern void shell_restore_init(RecoveryModuleCallbacks *cb); #endif /* XLOG_ARCHIVE_H */ diff --git a/src/include/access/xlogrecovery.h b/src/include/access/xlogrecovery.h index 47c29350f5..35d1d09374 100644 --- a/src/include/access/xlogrecovery.h +++ b/src/include/access/xlogrecovery.h @@ -55,6 +55,7 @@ extern PGDLLIMPORT int recovery_min_apply_delay; extern PGDLLIMPORT char *PrimaryConnInfo; extern PGDLLIMPORT char *PrimarySlotName; extern PGDLLIMPORT char *recoveryRestoreCommand; +extern PGDLLIMPORT char *restoreLibrary; extern PGDLLIMPORT char *recoveryEndCommand; extern PGDLLIMPORT char *archiveCleanupCommand; diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index ba89d013e6..947597247f 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -404,6 +404,8 @@ extern void *guc_malloc(int elevel, size_t size); extern pg_nodiscard void *guc_realloc(int elevel, void *old, size_t size); extern char *guc_strdup(int elevel, const char *src); extern void guc_free(void *ptr); +extern void CheckMutuallyExclusiveGUCs(const char *p1val, const char *p1name, + const char *p2val, const char *p2name); #ifdef EXEC_BACKEND extern void write_nondefault_variables(GucContext context); -- 2.25.1 --PEIAKu/WMn1b1Hv9-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-01-21 21:26 Jeff Davis <[email protected]> 0 siblings, 2 replies; 9+ messages in thread From: Jeff Davis @ 2025-01-21 21:26 UTC (permalink / raw) To: James Hunter <[email protected]>; [email protected] <[email protected]> On Fri, 2025-01-10 at 10:00 -0800, James Hunter wrote: > How should “query_work_mem” work? Let’s start with an example: > suppose > we have an OLAP query that has 2 Hash Joins, and no other operators > that use work_mem. So we plan first, and then assign available memory afterward? If we do it that way, then the costing will be inaccurate, because the original costs are based on the original work_mem. It may be better than killing the query, but not ideal. > -- Second, we could say, instead: the small Hash Join is *highly > unlikely* to use > 1 MB, so let’s just give both Hash Joins 1023 MB, > expecting that the small Hash Join won’t use more than 1 MB of its > 1023 MB allotment anyway, so we won’t run OOM. In effect, we’re > oversubscribing, betting that the small Hash Join will just stay > within some smaller, “unenforced” memory limit. > > In this example, this bet is probably fine — but it won’t work in > general. I don’t want to be in the business of gambling with customer > resources: if the small Hash Join is unlikely to use more than 1 MB, > then let’s just assign it 1 MB of work_mem. I like this idea. Operators that either know they don't need much memory, or estimate that they don't need much memory, can constrain themselves. That would protect against misestimations and advertise to the higher levels of the planner how much memory the operator actually wants. Right now, the planner doesn't know which operators need a lot of memory and which ones don't need any significant amount at all. The challenge, of course, is what the higher levels of the planner would do with that information, which goes to the rest of your proposal. But tracking the information seems very reasonable to me. > I propose that we add a “query_work_mem” GUC, which works by > distributing (using some algorithm to be described in a follow-up > email) the entire “query_work_mem” to the query’s operators. And then > each operator will spill when it exceeds its own work_mem limit. So > we’ll preserve the existing “spill” logic as much as possible. The description above sounds too "top-down" to me. That may work, but has the disadvantage that costing has already happened. We should also consider: * Reusing the path generation infrastructure so that both "high memory" and "low memory" paths can be considered, and if a path requires too much memory in aggregate, then it would be rejected in favor of a path that uses less memory. This feels like it fits within the planner architecture the best, but it also might lead to a path explosion, so we may need additional controls. * Some kind of negotiation where the top level of the planner finds that the plan uses too much memory, and replans some or all of it. (I think is similar to what you described as the "feedback loop" later in your email.) I agree that this is complex and may not have enough benefit to justify. Regards, Jeff Davis ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-01-22 20:48 Tomas Vondra <[email protected]> parent: Jeff Davis <[email protected]> 1 sibling, 0 replies; 9+ messages in thread From: Tomas Vondra @ 2025-01-22 20:48 UTC (permalink / raw) To: Jeff Davis <[email protected]>; James Hunter <[email protected]>; [email protected] <[email protected]> On 1/21/25 22:26, Jeff Davis wrote: > On Fri, 2025-01-10 at 10:00 -0800, James Hunter wrote: >> How should “query_work_mem” work? Let’s start with an example: >> suppose >> we have an OLAP query that has 2 Hash Joins, and no other operators >> that use work_mem. > > So we plan first, and then assign available memory afterward? If we do > it that way, then the costing will be inaccurate, because the original > costs are based on the original work_mem. > > It may be better than killing the query, but not ideal. > >> -- Second, we could say, instead: the small Hash Join is *highly >> unlikely* to use > 1 MB, so let’s just give both Hash Joins 1023 MB, >> expecting that the small Hash Join won’t use more than 1 MB of its >> 1023 MB allotment anyway, so we won’t run OOM. In effect, we’re >> oversubscribing, betting that the small Hash Join will just stay >> within some smaller, “unenforced” memory limit. >> >> In this example, this bet is probably fine — but it won’t work in >> general. I don’t want to be in the business of gambling with customer >> resources: if the small Hash Join is unlikely to use more than 1 MB, >> then let’s just assign it 1 MB of work_mem. > > I like this idea. Operators that either know they don't need much > memory, or estimate that they don't need much memory, can constrain > themselves. That would protect against misestimations and advertise to > the higher levels of the planner how much memory the operator actually > wants. Right now, the planner doesn't know which operators need a lot > of memory and which ones don't need any significant amount at all. > I'm not sure I like the idea that much. At first restricting the operator to the amount the optimizer predicts will be needed seems reasonable, because that's generally the best idea of memory usage we have without running the query. But these estimates are often pretty fundamentally unreliable - maybe not for simple examples, but once you put an aggregate on top of a join, the errors can be pretty wild. And allowing the operator to still use more work_mem makes this more adaptive. I suspect forcing the operator to adhere to the estimated work_mem might make this much worse (but I haven't tried, maybe spilling to temp files is not that bad). > The challenge, of course, is what the higher levels of the planner > would do with that information, which goes to the rest of your > proposal. But tracking the information seems very reasonable to me. > I agree. Tracking additional information seems like a good idea, but it's not clear to me what would the planner use this. I can imagine various approaches - e.g. we might do the planning as usual and then distribute the query_work_mem between the nodes in proportion to the estimated amount of memory. But it all seems like a very ad hoc heuristics, and easy to confuse / make the wrong decision. >> I propose that we add a “query_work_mem” GUC, which works by >> distributing (using some algorithm to be described in a follow-up >> email) the entire “query_work_mem” to the query’s operators. And then >> each operator will spill when it exceeds its own work_mem limit. So >> we’ll preserve the existing “spill” logic as much as possible. > > The description above sounds too "top-down" to me. That may work, but > has the disadvantage that costing has already happened. We should also > consider: > > * Reusing the path generation infrastructure so that both "high memory" > and "low memory" paths can be considered, and if a path requires too > much memory in aggregate, then it would be rejected in favor of a path > that uses less memory. This feels like it fits within the planner > architecture the best, but it also might lead to a path explosion, so > we may need additional controls. > > * Some kind of negotiation where the top level of the planner finds > that the plan uses too much memory, and replans some or all of it. (I > think is similar to what you described as the "feedback loop" later in > your email.) I agree that this is complex and may not have enough > benefit to justify. > Right, it seems rather at odds with the bottom-up construction of paths. The amount of memory an operator may use seems like a pretty fundamental information, but if it's available only after the whole plan is built, that seems ... not great. I don't know if generating (and keeping) low/high-memory paths is quite feasible. Isn't that really a continuum for many paths? A hash join may need very little memory (with batching) or a lot of memory (if keeping everything in memory), so how would this work? Would we generate paths for a range of work_mem values (with different costs)? regards -- Tomas Vondra ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-01-25 01:04 James Hunter <[email protected]> parent: Jeff Davis <[email protected]> 1 sibling, 1 reply; 9+ messages in thread From: James Hunter @ 2025-01-25 01:04 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: [email protected] <[email protected]> On Tue, Jan 21, 2025 at 1:26 PM Jeff Davis <[email protected]> wrote: > > On Fri, 2025-01-10 at 10:00 -0800, James Hunter wrote: > > How should “query_work_mem” work? Let’s start with an example: > > suppose > > we have an OLAP query that has 2 Hash Joins, and no other operators > > that use work_mem. > > So we plan first, and then assign available memory afterward? If we do > it that way, then the costing will be inaccurate, because the original > costs are based on the original work_mem. > > It may be better than killing the query, but not ideal. As you point out, the outcome is better, but not ideal. My intuition is that an "ideal" solution would increase query compilation times beyond what customers would accept... But at least the outcome, if not ideal is better than killing the query! So it is a net improvement. > > I propose that we add a “query_work_mem” GUC, which works by > > distributing (using some algorithm to be described in a follow-up > > email) the entire “query_work_mem” to the query’s operators. And then > > each operator will spill when it exceeds its own work_mem limit. So > > we’ll preserve the existing “spill” logic as much as possible. > > The description above sounds too "top-down" to me. That may work, but > has the disadvantage that costing has already happened. We should also > consider: > > * Reusing the path generation infrastructure so that both "high memory" > and "low memory" paths can be considered, and if a path requires too > much memory in aggregate, then it would be rejected in favor of a path > that uses less memory. This feels like it fits within the planner > architecture the best, but it also might lead to a path explosion, so > we may need additional controls. > > * Some kind of negotiation where the top level of the planner finds > that the plan uses too much memory, and replans some or all of it. (I > think is similar to what you described as the "feedback loop" later in > your email.) I agree that this is complex and may not have enough > benefit to justify. Generating "high memory" vs. "low memory" paths would be tricky, because the definition of "high" vs. "low" depends on the entire path tree, not just on a single path node. So I think it would quickly lead to a state-space explosion, as you mention. And I think negotiation has the same problem: it's based on the entire tree, not just an individual path node. I think the general problem is not so much "top-down" vs. "bottom-up", as "individual path node" vs. "entire path tree." Today, PostgreSQL costs each path node individually, by referring to the static "work_mem" GUC. In any attempt to improve the optimizer's choice, I think we'd have to cost the entire path tree. And there are many more trees than there are tree nodes. For example, the decision whether to prefer a Nested Loop vs. a Hash Join that takes 2 MB of working memory, depends on what the query's other joins are doing. At any rate, I think we can solve the problem of "killing the query" now; and then worry, in the future, about the ideal solution of how to pick the optimal execution plan. > Regards, > Jeff Davis Thanks for your comments! James ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-01-25 01:48 Jeff Davis <[email protected]> parent: James Hunter <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Jeff Davis @ 2025-01-25 01:48 UTC (permalink / raw) To: James Hunter <[email protected]>; +Cc: [email protected] <[email protected]> On Fri, 2025-01-24 at 17:04 -0800, James Hunter wrote: > Generating "high memory" vs. "low memory" paths would be tricky, > because the definition of "high" vs. "low" depends on the entire path > tree, not just on a single path node. So I think it would quickly > lead > to a state-space explosion, as you mention. At first, it appears to lead to an explosion, but there are a lot of ways to prune early. Many operators, like an index scan, don't even need to track memory, so they'd just have the one path. Other operators can just generate a low memory path because estimates show that it's unlikely to need more than that. And if there's a blocking operator, then that resets the memory requirement, pruning the space further. And I assume you are talking about analytic queries with reasonably large values of work_mem anyway. That justifies a bit more planning time -- no need to generate extra paths for cheap queries. Maybe my idea doesn't work out, but I think it's too early to dismiss it. Regards, Jeff Davis ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-02-11 03:09 James Hunter <[email protected]> parent: Jeff Davis <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: James Hunter @ 2025-02-11 03:09 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: [email protected] <[email protected]> On Fri, Jan 24, 2025 at 5:48 PM Jeff Davis <[email protected]> wrote: > > On Fri, 2025-01-24 at 17:04 -0800, James Hunter wrote: > > Generating "high memory" vs. "low memory" paths would be tricky, > > because the definition of "high" vs. "low" depends on the entire path > > tree, not just on a single path node. So I think it would quickly > > lead > > to a state-space explosion, as you mention. > > At first, it appears to lead to an explosion, but there are a lot of > ways to prune early. ... > > Maybe my idea doesn't work out, but I think it's too early to dismiss > it. I think it makes sense to split the work into two parts: one part that improves SQL execution, and a second part that improves the optimizer, to reflect the improvements to execution. It seems better to me to wait until we have the ability to enforce memory limits, before worrying about ways to generate different paths with different memory limits. Then we would be able to tune the optimizer heuristics based on the actual executor, instead of extrapolating how the executor would behave under different memory limits. James ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-02-11 18:00 Jeff Davis <[email protected]> parent: James Hunter <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: Jeff Davis @ 2025-02-11 18:00 UTC (permalink / raw) To: James Hunter <[email protected]>; +Cc: [email protected] <[email protected]> On Mon, 2025-02-10 at 19:09 -0800, James Hunter wrote: > I think it makes sense to split the work into two parts: one part > that > improves SQL execution, and a second part that improves the > optimizer, > to reflect the improvements to execution. I like the idea to store the value of work_mem in the path/plan/executor nodes, and use that at execution time rather than the GUC directly. IIUC, that would allow an extension to do what you want, right? A planner hook could just walk the tree and edit those values for individual nodes, and the executor would enforce them. Regards, Jeff Davis ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-02-11 18:39 James Hunter <[email protected]> parent: Jeff Davis <[email protected]> 0 siblings, 1 reply; 9+ messages in thread From: James Hunter @ 2025-02-11 18:39 UTC (permalink / raw) To: Jeff Davis <[email protected]>; +Cc: [email protected] <[email protected]> On Tue, Feb 11, 2025 at 10:00 AM Jeff Davis <[email protected]> wrote: > > On Mon, 2025-02-10 at 19:09 -0800, James Hunter wrote: > > I think it makes sense to split the work into two parts: one part > > that > > improves SQL execution, and a second part that improves the > > optimizer, > > to reflect the improvements to execution. > > I like the idea to store the value of work_mem in the > path/plan/executor nodes, and use that at execution time rather than > the GUC directly. > > IIUC, that would allow an extension to do what you want, right? A > planner hook could just walk the tree and edit those values for > individual nodes, and the executor would enforce them. Yes, exactly! * The Path would store "nbytes" (= the optimizer's estimate of how much working memory a given Path will use), to allow for future optimizer logic to consider memory usage when choosing the best Path. * The Plan would store a copy of "nbytes," along with "work_mem," and the executor would enforce work_mem. A "(work_mem on)" option to the "EXPLAIN" command would display both "nbytes" and "work_mem", per Plan node. * Either built-in logic or an extensibility hook would set "work_mem" on each individual Plan node, based on whatever heuristic or rule it chooses. Right now, my prototype sets "work_mem" inside ExecInitNode(). Thanks, James ^ permalink raw reply [nested|flat] 9+ messages in thread
* Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators @ 2025-02-11 22:04 Jeff Davis <[email protected]> parent: James Hunter <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jeff Davis @ 2025-02-11 22:04 UTC (permalink / raw) To: James Hunter <[email protected]>; +Cc: [email protected] <[email protected]> On Tue, 2025-02-11 at 10:39 -0800, James Hunter wrote: > * The Path would store "nbytes" (= the optimizer's estimate of how > much working memory a given Path will use), to allow for future > optimizer logic to consider memory usage when choosing the best Path. > > * The Plan would store a copy of "nbytes," along with "work_mem," and > the executor would enforce work_mem. A "(work_mem on)" option to the > "EXPLAIN" command would display both "nbytes" and "work_mem", per > Plan > node. Storing work_mem in each Plan node, and using that to enforce the memory limit (rather than using the GUC directly), seems uncontroversial to me. I'd suggest a standalone patch. Storing the optimizer's estimate of the memory wanted also sounds like a good idea. Let's pick a better name than "nbytes" though; maybe "requested_mem" or something? This change would make it a lot easier for an extension to adjust the per-node-work_mem, and also seems like good infrastructure for anything we build into the planner later. I suggest a standalone patch for this, as well. Can you write a useful extension with just the above two core patches? Regards, Jeff Davis ^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2025-02-11 22:04 UTC | newest] Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2022-12-10 03:40 [PATCH v7 2/2] Allow recovery via loadable modules. Nathan Bossart <[email protected]> 2025-01-21 21:26 Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators Jeff Davis <[email protected]> 2025-01-22 20:48 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators Tomas Vondra <[email protected]> 2025-01-25 01:04 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators James Hunter <[email protected]> 2025-01-25 01:48 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators Jeff Davis <[email protected]> 2025-02-11 03:09 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators James Hunter <[email protected]> 2025-02-11 18:00 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators Jeff Davis <[email protected]> 2025-02-11 18:39 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators James Hunter <[email protected]> 2025-02-11 22:04 ` Re: Proposal: "query_work_mem" GUC, to distribute working memory to the query's individual operators Jeff Davis <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox