agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v16 2/5] refactor code for restoring via shell 3+ messages / 2 participants [nested] [flat]
* [PATCH v16 2/5] refactor code for restoring via shell @ 2023-02-15 18:36 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Nathan Bossart @ 2023-02-15 18:36 UTC (permalink / raw) --- src/backend/Makefile | 2 +- src/backend/access/transam/timeline.c | 12 +- src/backend/access/transam/xlog.c | 50 ++++- src/backend/access/transam/xlogarchive.c | 167 ++++----------- src/backend/access/transam/xlogrecovery.c | 3 +- src/backend/meson.build | 1 + src/backend/postmaster/startup.c | 16 +- src/backend/restore/Makefile | 18 ++ src/backend/restore/meson.build | 5 + src/backend/restore/shell_restore.c | 245 ++++++++++++++++++++++ src/include/Makefile | 2 +- src/include/access/xlogarchive.h | 9 +- src/include/meson.build | 1 + src/include/postmaster/startup.h | 1 + src/include/restore/shell_restore.h | 26 +++ src/tools/pgindent/typedefs.list | 1 + 16 files changed, 407 insertions(+), 152 deletions(-) create mode 100644 src/backend/restore/Makefile create mode 100644 src/backend/restore/meson.build create mode 100644 src/backend/restore/shell_restore.c create mode 100644 src/include/restore/shell_restore.h diff --git a/src/backend/Makefile b/src/backend/Makefile index 3e275ac759..6913601c36 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = access archive backup bootstrap catalog parser commands executor \ foreign lib libpq \ main nodes optimizer partitioning port postmaster \ - regex replication rewrite \ + regex replication restore rewrite \ statistics storage tcop tsearch utils $(top_builddir)/src/timezone \ jit diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index 94e152694e..b3e1c18c65 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -59,7 +59,8 @@ restoreTimeLineHistoryFiles(TimeLineID begin, TimeLineID end) continue; TLHistoryFileName(histfname, tli); - if (RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false)) + if (RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false, + ARCHIVE_TYPE_TIMELINE_HISTORY)) KeepFileRestoredFromArchive(path, histfname); } } @@ -97,7 +98,8 @@ readTimeLineHistory(TimeLineID targetTLI) { TLHistoryFileName(histfname, targetTLI); fromArchive = - RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false); + RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false, + ARCHIVE_TYPE_TIMELINE_HISTORY); } else TLHistoryFilePath(path, targetTLI); @@ -232,7 +234,8 @@ existsTimeLineHistory(TimeLineID probeTLI) if (ArchiveRecoveryRequested) { TLHistoryFileName(histfname, probeTLI); - RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false); + RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false, + ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS); } else TLHistoryFilePath(path, probeTLI); @@ -334,7 +337,8 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, if (ArchiveRecoveryRequested) { TLHistoryFileName(histfname, parentTLI); - RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false); + RestoreArchivedFile(path, histfname, "RECOVERYHISTORY", 0, false, + ARCHIVE_TYPE_TIMELINE_HISTORY); } else TLHistoryFilePath(path, parentTLI); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 40461923ea..1307748b4e 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -84,6 +84,7 @@ #include "replication/snapbuild.h" #include "replication/walreceiver.h" #include "replication/walsender.h" +#include "restore/shell_restore.h" #include "storage/bufmgr.h" #include "storage/fd.h" #include "storage/ipc.h" @@ -702,6 +703,7 @@ static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli); static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos); static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos); static uint64 XLogRecPtrToBytePos(XLogRecPtr ptr); +static void GetOldestRestartPointFileName(char *fname); static void WALInsertLockAcquire(void); static void WALInsertLockAcquireExclusive(void); @@ -4982,12 +4984,18 @@ CleanupAfterArchiveRecovery(TimeLineID EndOfLogTLI, XLogRecPtr EndOfLog, { /* * Execute the recovery_end_command, if any. + * + * The command is provided the archive file cutoff point for use during + * log shipping replication. All files earlier than this point can be + * deleted from the archive, though there is no requirement to do so. */ - if (recoveryEndCommand && strcmp(recoveryEndCommand, "") != 0) - ExecuteRecoveryCommand(recoveryEndCommand, - "recovery_end_command", - true, - WAIT_EVENT_RECOVERY_END_COMMAND); + if (shell_recovery_end_configured()) + { + char lastRestartPointFname[MAXFNAMELEN]; + + GetOldestRestartPointFileName(lastRestartPointFname); + shell_recovery_end(lastRestartPointFname); + } /* * We switched to a new timeline. Clean up segments on the old timeline. @@ -7446,12 +7454,18 @@ CreateRestartPoint(int flags) /* * Finally, execute archive_cleanup_command, if any. + * + * The command is provided the archive file cutoff point for use during + * log shipping replication. All files earlier than this point can be + * deleted from the archive, though there is no requirement to do so. */ - if (archiveCleanupCommand && strcmp(archiveCleanupCommand, "") != 0) - ExecuteRecoveryCommand(archiveCleanupCommand, - "archive_cleanup_command", - false, - WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND); + if (shell_archive_cleanup_configured()) + { + char lastRestartPointFname[MAXFNAMELEN]; + + GetOldestRestartPointFileName(lastRestartPointFname); + shell_archive_cleanup(lastRestartPointFname); + } return true; } @@ -9067,6 +9081,22 @@ GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli) LWLockRelease(ControlFileLock); } +/* + * Returns the WAL file name for the last checkpoint or restartpoint. This is + * the oldest WAL file that we still need if we have to restart recovery. + */ +static void +GetOldestRestartPointFileName(char *fname) +{ + XLogRecPtr restartRedoPtr; + TimeLineID restartTli; + XLogSegNo restartSegNo; + + GetOldestRestartPoint(&restartRedoPtr, &restartTli); + XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size); + XLogFileName(fname, restartTli, restartSegNo, wal_segment_size); +} + /* Thin wrapper around ShutdownWalRcv(). */ void XLogShutdownWalRcv(void) diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c index 524e80adb1..a925ac22f6 100644 --- a/src/backend/access/transam/xlogarchive.c +++ b/src/backend/access/transam/xlogarchive.c @@ -23,12 +23,12 @@ #include "access/xlog_internal.h" #include "access/xlogarchive.h" #include "common/archive.h" -#include "common/percentrepl.h" #include "miscadmin.h" #include "pgstat.h" #include "postmaster/startup.h" #include "postmaster/pgarch.h" #include "replication/walsender.h" +#include "restore/shell_restore.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -54,12 +54,11 @@ bool RestoreArchivedFile(char *path, const char *xlogfname, const char *recovername, off_t expectedSize, - bool cleanupEnabled) + bool cleanupEnabled, ArchiveType archive_type) { char xlogpath[MAXPGPATH]; - char *xlogRestoreCmd; char lastRestartPointFname[MAXPGPATH]; - int rc; + bool ret = false; struct stat stat_buf; XLogSegNo restartSegNo; XLogRecPtr restartRedoPtr; @@ -73,8 +72,21 @@ RestoreArchivedFile(char *path, const char *xlogfname, goto not_available; /* In standby mode, restore_command might not be supplied */ - if (recoveryRestoreCommand == NULL || strcmp(recoveryRestoreCommand, "") == 0) - goto not_available; + switch (archive_type) + { + case ARCHIVE_TYPE_WAL_SEGMENT: + if (!shell_restore_file_configured()) + goto not_available; + break; + case ARCHIVE_TYPE_TIMELINE_HISTORY: + if (!shell_restore_file_configured()) + goto not_available; + break; + case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS: + if (!shell_restore_file_configured()) + goto not_available; + break; + } /* * When doing archive recovery, we always prefer an archived log file even @@ -150,39 +162,33 @@ RestoreArchivedFile(char *path, const char *xlogfname, else XLogFileName(lastRestartPointFname, 0, 0, wal_segment_size); - /* Build the restore command to execute */ - xlogRestoreCmd = BuildRestoreCommand(recoveryRestoreCommand, - xlogpath, xlogfname, - lastRestartPointFname); - - ereport(DEBUG3, - (errmsg_internal("executing restore command \"%s\"", - xlogRestoreCmd))); - - fflush(NULL); - pgstat_report_wait_start(WAIT_EVENT_RESTORE_COMMAND); - /* - * PreRestoreCommand() informs the SIGTERM handler for the startup process - * that it should proc_exit() right away. This is done for the duration - * of the system() call because there isn't a good way to break out while - * it is executing. Since we might call proc_exit() in a signal handler, - * it is best to put any additional logic before or after the - * PreRestoreCommand()/PostRestoreCommand() section. + * To ensure we are responsive to server shutdown, check for shutdown + * requests before and after restoring a file. If there is one, we exit + * right away. */ - PreRestoreCommand(); + HandleStartupProcShutdownRequests(); /* * Copy xlog from archival storage to XLOGDIR */ - rc = system(xlogRestoreCmd); - - PostRestoreCommand(); + switch (archive_type) + { + case ARCHIVE_TYPE_WAL_SEGMENT: + ret = shell_restore_wal_segment(xlogfname, xlogpath, + lastRestartPointFname); + break; + case ARCHIVE_TYPE_TIMELINE_HISTORY: + ret = shell_restore_timeline_history(xlogfname, xlogpath); + break; + case ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS: + ret = shell_restore_timeline_history(xlogfname, xlogpath); + break; + } - pgstat_report_wait_end(); - pfree(xlogRestoreCmd); + HandleStartupProcShutdownRequests(); - if (rc == 0) + if (ret) { /* * command apparently succeeded, but let's make sure the file is @@ -238,37 +244,6 @@ RestoreArchivedFile(char *path, const char *xlogfname, } } - /* - * Remember, we rollforward UNTIL the restore fails so failure here is - * just part of the process... that makes it difficult to determine - * whether the restore failed because there isn't an archive to restore, - * or because the administrator has specified the restore program - * incorrectly. We have to assume the former. - * - * However, if the failure was due to any sort of signal, it's best to - * punt and abort recovery. (If we "return false" here, upper levels will - * assume that recovery is complete and start up the database!) It's - * essential to abort on child SIGINT and SIGQUIT, because per spec - * system() ignores SIGINT and SIGQUIT while waiting; if we see one of - * those it's a good bet we should have gotten it too. - * - * On SIGTERM, assume we have received a fast shutdown request, and exit - * cleanly. It's pure chance whether we receive the SIGTERM first, or the - * child process. If we receive it first, the signal handler will call - * proc_exit, otherwise we do it here. If we or the child process received - * SIGTERM for any other reason than a fast shutdown request, postmaster - * will perform an immediate shutdown when it sees us exiting - * unexpectedly. - * - * We treat hard shell errors such as "command not found" as fatal, too. - */ - if (wait_result_is_signal(rc, SIGTERM)) - proc_exit(1); - - ereport(wait_result_is_any_signal(rc, true) ? FATAL : DEBUG2, - (errmsg("could not restore file \"%s\" from archive: %s", - xlogfname, wait_result_to_str(rc)))); - not_available: /* @@ -282,74 +257,6 @@ not_available: return false; } -/* - * Attempt to execute an external shell command during recovery. - * - * 'command' is the shell command to be executed, 'commandName' is a - * human-readable name describing the command emitted in the logs. If - * 'failOnSignal' is true and the command is killed by a signal, a FATAL - * error is thrown. Otherwise a WARNING is emitted. - * - * This is currently used for recovery_end_command and archive_cleanup_command. - */ -void -ExecuteRecoveryCommand(const char *command, const char *commandName, - bool failOnSignal, uint32 wait_event_info) -{ - char *xlogRecoveryCmd; - char lastRestartPointFname[MAXPGPATH]; - int rc; - XLogSegNo restartSegNo; - XLogRecPtr restartRedoPtr; - TimeLineID restartTli; - - Assert(command && commandName); - - /* - * Calculate the archive file cutoff point for use during log shipping - * replication. All files earlier than this point can be deleted from the - * archive, though there is no requirement to do so. - */ - GetOldestRestartPoint(&restartRedoPtr, &restartTli); - XLByteToSeg(restartRedoPtr, restartSegNo, wal_segment_size); - XLogFileName(lastRestartPointFname, restartTli, restartSegNo, - wal_segment_size); - - /* - * construct the command to be executed - */ - xlogRecoveryCmd = replace_percent_placeholders(command, commandName, "r", lastRestartPointFname); - - ereport(DEBUG3, - (errmsg_internal("executing %s \"%s\"", commandName, command))); - - /* - * execute the constructed command - */ - fflush(NULL); - pgstat_report_wait_start(wait_event_info); - rc = system(xlogRecoveryCmd); - pgstat_report_wait_end(); - - pfree(xlogRecoveryCmd); - - if (rc != 0) - { - /* - * If the failure was due to any sort of signal, it's best to punt and - * abort recovery. See comments in RestoreArchivedFile(). - */ - ereport((failOnSignal && wait_result_is_any_signal(rc, true)) ? FATAL : WARNING, - /*------ - translator: First %s represents a postgresql.conf parameter name like - "recovery_end_command", the 2nd is the value of that parameter, the - third an already translated error message. */ - (errmsg("%s \"%s\": %s", commandName, - command, wait_result_to_str(rc)))); - } -} - - /* * A file was restored from the archive under a temporary filename (path), * and now we want to keep it. Rename it under the permanent filename in diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 315e4b27cb..d43c229eca 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -4142,7 +4142,8 @@ XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli, if (!RestoreArchivedFile(path, xlogfname, "RECOVERYXLOG", wal_segment_size, - InRedo)) + InRedo, + ARCHIVE_TYPE_WAL_SEGMENT)) return -1; break; diff --git a/src/backend/meson.build b/src/backend/meson.build index 88a35e9676..0381bae024 100644 --- a/src/backend/meson.build +++ b/src/backend/meson.build @@ -27,6 +27,7 @@ subdir('port') subdir('postmaster') subdir('regex') subdir('replication') +subdir('restore') subdir('rewrite') subdir('statistics') subdir('storage') diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index 0e7de26bc2..adce0ffcef 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -198,8 +198,7 @@ HandleStartupProcInterrupts(void) /* * Check if we were requested to exit without finishing recovery. */ - if (shutdown_requested) - proc_exit(1); + HandleStartupProcShutdownRequests(); /* * Emergency bailout if postmaster has died. This is to avoid the @@ -223,6 +222,16 @@ HandleStartupProcInterrupts(void) ProcessLogMemoryContextInterrupt(); } +/* + * If there is a pending shutdown request, exit. + */ +void +HandleStartupProcShutdownRequests(void) +{ + if (shutdown_requested) + proc_exit(1); +} + /* -------------------------------- * signal handler routines @@ -298,8 +307,7 @@ PreRestoreCommand(void) * shutdown request received just before this. */ in_restore_command = true; - if (shutdown_requested) - proc_exit(1); + HandleStartupProcShutdownRequests(); } void diff --git a/src/backend/restore/Makefile b/src/backend/restore/Makefile new file mode 100644 index 0000000000..983d8e980f --- /dev/null +++ b/src/backend/restore/Makefile @@ -0,0 +1,18 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for src/backend/restore +# +# IDENTIFICATION +# src/backend/restore/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/restore +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + shell_restore.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/restore/meson.build b/src/backend/restore/meson.build new file mode 100644 index 0000000000..2b27cfced0 --- /dev/null +++ b/src/backend/restore/meson.build @@ -0,0 +1,5 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group + +backend_sources += files( + 'shell_restore.c' +) diff --git a/src/backend/restore/shell_restore.c b/src/backend/restore/shell_restore.c new file mode 100644 index 0000000000..bc1f4a431a --- /dev/null +++ b/src/backend/restore/shell_restore.c @@ -0,0 +1,245 @@ +/*------------------------------------------------------------------------- + * + * shell_restore.c + * + * These restore functions use a user-specified shell command (e.g., the + * restore_command GUC). + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/restore/shell_restore.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include <signal.h> + +#include "access/xlog.h" +#include "access/xlogrecovery.h" +#include "access/xlog_internal.h" +#include "common/archive.h" +#include "common/percentrepl.h" +#include "postmaster/startup.h" +#include "restore/shell_restore.h" +#include "storage/ipc.h" +#include "utils/wait_event.h" + +static bool shell_restore_file(const char *file, const char *path, + const char *lastRestartPointFileName); +static void ExecuteRecoveryCommand(const char *command, + const char *commandName, + bool failOnSignal, + uint32 wait_event_info, + const char *lastRestartPointFileName); + +/* + * Attempt to execute a shell-based restore command to retrieve a WAL segment. + * + * Returns true if the command has succeeded, false otherwise. + */ +bool +shell_restore_wal_segment(const char *file, const char *path, + const char *lastRestartPointFileName) +{ + return shell_restore_file(file, path, lastRestartPointFileName); +} + +/* + * Attempt to execute a shell-based restore command to retrieve a timeline + * history file. + * + * Returns true if the command has succeeded, false otherwise. + */ +bool +shell_restore_timeline_history(const char *file, const char *path) +{ + char lastRestartPointFname[MAXPGPATH]; + + XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size); + return shell_restore_file(file, path, lastRestartPointFname); +} + +/* + * Check whether restore_command is supplied. + */ +bool +shell_restore_file_configured(void) +{ + return recoveryRestoreCommand[0] != '\0'; +} + +/* + * Attempt to execute a shell-based restore command to retrieve a file. + * + * Returns true if the command has succeeded, false otherwise. + */ +static bool +shell_restore_file(const char *file, const char *path, + const char *lastRestartPointFileName) +{ + char *cmd; + int rc; + + /* Build the restore command to execute */ + cmd = BuildRestoreCommand(recoveryRestoreCommand, path, file, + lastRestartPointFileName); + + ereport(DEBUG3, + (errmsg_internal("executing restore command \"%s\"", cmd))); + + fflush(NULL); + pgstat_report_wait_start(WAIT_EVENT_RESTORE_COMMAND); + + /* + * PreRestoreCommand() informs the SIGTERM handler for the startup process + * that it should proc_exit() right away. This is done for the duration + * of the system() call because there isn't a good way to break out while + * it is executing. Since we might call proc_exit() in a signal handler, + * it is best to put any additional logic before or after the + * PreRestoreCommand()/PostRestoreCommand() section. + */ + PreRestoreCommand(); + + /* + * Copy xlog from archival storage to XLOGDIR + */ + rc = system(cmd); + + PostRestoreCommand(); + + pgstat_report_wait_end(); + pfree(cmd); + + /* + * Remember, we rollforward UNTIL the restore fails so failure here is + * just part of the process... that makes it difficult to determine + * whether the restore failed because there isn't an archive to restore, + * or because the administrator has specified the restore program + * incorrectly. We have to assume the former. + * + * However, if the failure was due to any sort of signal, it's best to + * punt and abort recovery. (If we "return false" here, upper levels will + * assume that recovery is complete and start up the database!) It's + * essential to abort on child SIGINT and SIGQUIT, because per spec + * system() ignores SIGINT and SIGQUIT while waiting; if we see one of + * those it's a good bet we should have gotten it too. + * + * On SIGTERM, assume we have received a fast shutdown request, and exit + * cleanly. It's pure chance whether we receive the SIGTERM first, or the + * child process. If we receive it first, the signal handler will call + * proc_exit, otherwise we do it here. If we or the child process received + * SIGTERM for any other reason than a fast shutdown request, postmaster + * will perform an immediate shutdown when it sees us exiting + * unexpectedly. + * + * We treat hard shell errors such as "command not found" as fatal, too. + */ + if (rc != 0) + { + if (wait_result_is_signal(rc, SIGTERM)) + proc_exit(1); + + ereport(wait_result_is_any_signal(rc, true) ? FATAL : DEBUG2, + (errmsg("could not restore file \"%s\" from archive: %s", + file, wait_result_to_str(rc)))); + } + + return (rc == 0); +} + +/* + * Check whether archive_cleanup_command is supplied. + */ +bool +shell_archive_cleanup_configured(void) +{ + return archiveCleanupCommand[0] != '\0'; +} + +/* + * Attempt to execute a shell-based archive cleanup command. + */ +void +shell_archive_cleanup(const char *lastRestartPointFileName) +{ + ExecuteRecoveryCommand(archiveCleanupCommand, "archive_cleanup_command", + false, WAIT_EVENT_ARCHIVE_CLEANUP_COMMAND, + lastRestartPointFileName); +} + +/* + * Check whether recovery_end_command is supplied. + */ +bool +shell_recovery_end_configured(void) +{ + return recoveryEndCommand[0] != '\0'; +} + +/* + * Attempt to execute a shell-based end-of-recovery command. + */ +void +shell_recovery_end(const char *lastRestartPointFileName) +{ + ExecuteRecoveryCommand(recoveryEndCommand, "recovery_end_command", true, + WAIT_EVENT_RECOVERY_END_COMMAND, + lastRestartPointFileName); +} + +/* + * Attempt to execute an external shell command during recovery. + * + * 'command' is the shell command to be executed, 'commandName' is a + * human-readable name describing the command emitted in the logs. If + * 'failOnSignal' is true and the command is killed by a signal, a FATAL + * error is thrown. Otherwise a WARNING is emitted. + * + * This is currently used for recovery_end_command and archive_cleanup_command. + */ +static void +ExecuteRecoveryCommand(const char *command, const char *commandName, + bool failOnSignal, uint32 wait_event_info, + const char *lastRestartPointFileName) +{ + char *xlogRecoveryCmd; + int rc; + + Assert(command && commandName); + + /* + * construct the command to be executed + */ + xlogRecoveryCmd = replace_percent_placeholders(command, commandName, "r", + lastRestartPointFileName); + + ereport(DEBUG3, + (errmsg_internal("executing %s \"%s\"", commandName, command))); + + /* + * execute the constructed command + */ + fflush(NULL); + pgstat_report_wait_start(wait_event_info); + rc = system(xlogRecoveryCmd); + pgstat_report_wait_end(); + + pfree(xlogRecoveryCmd); + + if (rc != 0) + { + /* + * If the failure was due to any sort of signal, it's best to punt and + * abort recovery. See comments in shell_restore(). + */ + ereport((failOnSignal && wait_result_is_any_signal(rc, true)) ? FATAL : WARNING, + /*------ + translator: First %s represents a postgresql.conf parameter name like + "recovery_end_command", the 2nd is the value of that parameter, the + third an already translated error message. */ + (errmsg("%s \"%s\": %s", commandName, + command, wait_result_to_str(rc)))); + } +} diff --git a/src/include/Makefile b/src/include/Makefile index a7ca277803..3d276d55aa 100644 --- a/src/include/Makefile +++ b/src/include/Makefile @@ -20,7 +20,7 @@ all: pg_config.h pg_config_ext.h pg_config_os.h SUBDIRS = access archive bootstrap catalog commands common datatype \ executor fe_utils foreign jit \ lib libpq mb nodes optimizer parser partitioning postmaster \ - regex replication rewrite \ + regex replication restore rewrite \ statistics storage tcop snowball snowball/libstemmer tsearch \ tsearch/dicts utils port port/atomics port/win32 port/win32_msvc \ port/win32_msvc/sys port/win32/arpa port/win32/netinet \ diff --git a/src/include/access/xlogarchive.h b/src/include/access/xlogarchive.h index 31ff206034..b9bd0a93cd 100644 --- a/src/include/access/xlogarchive.h +++ b/src/include/access/xlogarchive.h @@ -17,9 +17,16 @@ #include "access/xlogdefs.h" +typedef enum ArchiveType +{ + ARCHIVE_TYPE_WAL_SEGMENT, + ARCHIVE_TYPE_TIMELINE_HISTORY, + ARCHIVE_TYPE_TIMELINE_HISTORY_EXISTS +} ArchiveType; + extern bool RestoreArchivedFile(char *path, const char *xlogfname, const char *recovername, off_t expectedSize, - bool cleanupEnabled); + bool cleanupEnabled, ArchiveType archive_type); extern void ExecuteRecoveryCommand(const char *command, const char *commandName, bool failOnSignal, uint32 wait_event_info); extern void KeepFileRestoredFromArchive(const char *path, const char *xlogfname); diff --git a/src/include/meson.build b/src/include/meson.build index d50897c9fd..849e1e4dbf 100644 --- a/src/include/meson.build +++ b/src/include/meson.build @@ -150,6 +150,7 @@ header_subdirs = [ 'postmaster', 'regex', 'replication', + 'restore', 'rewrite', 'statistics', 'storage', diff --git a/src/include/postmaster/startup.h b/src/include/postmaster/startup.h index 6a2e4c4526..09d3f89c69 100644 --- a/src/include/postmaster/startup.h +++ b/src/include/postmaster/startup.h @@ -26,6 +26,7 @@ extern PGDLLIMPORT int log_startup_progress_interval; extern void HandleStartupProcInterrupts(void); +extern void HandleStartupProcShutdownRequests(void); extern void StartupProcessMain(void) pg_attribute_noreturn(); extern void PreRestoreCommand(void); extern void PostRestoreCommand(void); diff --git a/src/include/restore/shell_restore.h b/src/include/restore/shell_restore.h new file mode 100644 index 0000000000..570588e493 --- /dev/null +++ b/src/include/restore/shell_restore.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * shell_restore.h + * Exports for restoring via shell. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * src/include/restore/shell_restore.h + * + *------------------------------------------------------------------------- + */ +#ifndef _SHELL_RESTORE_H +#define _SHELL_RESTORE_H + +extern bool shell_restore_file_configured(void); +extern bool shell_restore_wal_segment(const char *file, const char *path, + const char *lastRestartPointFileName); +extern bool shell_restore_timeline_history(const char *file, const char *path); + +extern bool shell_archive_cleanup_configured(void); +extern void shell_archive_cleanup(const char *lastRestartPointFileName); + +extern bool shell_recovery_end_configured(void); +extern void shell_recovery_end(const char *lastRestartPointFileName); + +#endif /* _SHELL_RESTORE_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 06b25617bc..16220cfe44 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -134,6 +134,7 @@ ArchiveOpts ArchiveShutdownCB ArchiveStartupCB ArchiveStreamState +ArchiveType ArchiverOutput ArchiverStage ArrayAnalyzeExtraData -- 2.25.1 --SLDf9lqlvOQaIe6s Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0003-rename-archive-modules.sgml-to-archive-and-resto.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. @ 2026-06-22 07:34 Antonin Houska <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Antonin Houska @ 2026-06-22 07:34 UTC (permalink / raw) The default values may be needed by the executor when processing the concurrent data changes. In particular, ExecInsertIndexTuples() needs it when determining the value of the 'indexUnchanged' hint for the index AM. Like in copy_index_constraints(), we make the new catalog entries dependent on the transient relation, so they are dropped along with it automatically. --- src/backend/commands/repack.c | 97 +++++++++++++++++++ .../injection_points/specs/repack.spec | 3 +- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 4d177c868bb..725efa236bc 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -48,6 +48,7 @@ #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" +#include "catalog/pg_attrdef.h" #include "catalog/pg_constraint.h" #include "catalog/pg_inherits.h" #include "catalog/toasting.h" @@ -204,6 +205,7 @@ static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHea static List *build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes); static void copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id); +static void copy_attribute_defaults(Relation old_heap, Relation new_heap); static Relation process_single_relation(RepackStmt *stmt, LOCKMODE lockmode, bool isTopLevel, @@ -1083,6 +1085,13 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose, Assert(CheckRelationOidLockedByMe(OIDNewHeap, AccessExclusiveLock, false)); NewHeap = table_open(OIDNewHeap, NoLock); + /* + * Copy attribute defaults - the executor may need them, in order to + * process the concurrent data changes. In particular, this is related to + * ExecInsertIndexTuples(). + */ + copy_attribute_defaults(OldHeap, NewHeap); + /* Copy the heap data into the new table in the desired order */ copy_table_data(NewHeap, OldHeap, index, snapshot, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); @@ -3434,6 +3443,94 @@ copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id) CommandCounterIncrement(); } +/* + * Create a transient copy of attribute defaults for the transient table. + * + * Like above, the executor needs information on attribute defaults. Once the + * repacking is finished, the catalog entries we create here are dropped. + */ +static void +copy_attribute_defaults(Relation old_heap, Relation new_heap) +{ + Oid old_heap_id = RelationGetRelid(old_heap); + Oid new_heap_id = RelationGetRelid(new_heap); + ScanKeyData skey; + Relation rel; + Relation att_rel = NULL; + TupleDesc desc; + SysScanDesc scan; + HeapTuple tup; + ObjectAddress objrel; + + rel = table_open(AttrDefaultRelationId, RowExclusiveLock); + ObjectAddressSet(objrel, RelationRelationId, new_heap_id); + + ScanKeyInit(&skey, + Anum_pg_attrdef_adrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(old_heap_id)); + scan = systable_beginscan(rel, AttrDefaultIndexId, true, + NULL, 1, &skey); + desc = RelationGetDescr(rel); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_attrdef adform; + Oid oid; + Datum values[Natts_pg_attrdef] = {0}; + bool nulls[Natts_pg_attrdef] = {0}; + bool replaces[Natts_pg_attrdef] = {0}; + HeapTuple new_tup, att_tup, att_new_tup; + ObjectAddress objad; + Datum att_values[Natts_pg_attribute] = {0}; + bool att_nulls[Natts_pg_attribute] = {0}; + bool att_replaces[Natts_pg_attribute] = {0}; + + adform = (Form_pg_attrdef) GETSTRUCT(tup); + Assert(adform->adrelid == old_heap_id); + + oid = GetNewOidWithIndex(rel, AttrDefaultOidIndexId, + Anum_pg_attrdef_oid); + values[Anum_pg_attrdef_oid - 1] = ObjectIdGetDatum(oid); + replaces[Anum_pg_attrdef_oid - 1] = true; + values[Anum_pg_attrdef_adrelid - 1] = ObjectIdGetDatum(new_heap_id); + replaces[Anum_pg_attrdef_adrelid - 1] = true; + + new_tup = heap_modify_tuple(tup, desc, values, nulls, replaces); + + /* Insert it into the catalog. */ + CatalogTupleInsert(rel, new_tup); + + /* Create a dependency so it's removed when we drop the new heap. */ + ObjectAddressSet(objad, AttrDefaultRelationId, oid); + recordDependencyOn(&objad, &objrel, DEPENDENCY_AUTO); + + /* Set atthasdef - new heap has it cleared. */ + att_tup = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(new_heap_id), + ObjectIdGetDatum(adform->adnum)); + if (!HeapTupleIsValid(att_tup)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + adform->adnum, new_heap_id); + + att_values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(true); + att_replaces[Anum_pg_attribute_atthasdef - 1] = true; + + if (att_rel == NULL) + att_rel = table_open(AttributeRelationId, RowExclusiveLock); + att_new_tup = heap_modify_tuple(att_tup, RelationGetDescr(att_rel), + att_values, att_nulls, att_replaces); + ReleaseSysCache(att_tup); + CatalogTupleUpdate(att_rel, &att_new_tup->t_self, att_new_tup); + } + systable_endscan(scan); + + table_close(rel, RowExclusiveLock); + if (att_rel) + table_close(att_rel, RowExclusiveLock); + + CommandCounterIncrement(); +} + /* * Try to start a background worker to perform logical decoding of data * changes applied to relation while REPACK CONCURRENTLY is copying its diff --git a/src/test/modules/injection_points/specs/repack.spec b/src/test/modules/injection_points/specs/repack.spec index d727a9b056b..7896d1456ad 100644 --- a/src/test/modules/injection_points/specs/repack.spec +++ b/src/test/modules/injection_points/specs/repack.spec @@ -3,7 +3,8 @@ setup { CREATE EXTENSION injection_points; - CREATE TABLE repack_test(i int PRIMARY KEY, j int); + CREATE TABLE repack_test(i int PRIMARY KEY, j int, + k int GENERATED ALWAYS AS (j * 2) STORED); INSERT INTO repack_test(i, j) VALUES (1, 1), (2, 2), (3, 3), (4, 4); CREATE TABLE relfilenodes(node oid); -- 2.52.0 --=-=-=-- ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. @ 2026-06-22 07:34 Antonin Houska <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Antonin Houska @ 2026-06-22 07:34 UTC (permalink / raw) The default values may be needed by the executor when processing the concurrent data changes. In particular, ExecInsertIndexTuples() needs it when determining the value of the 'indexUnchanged' hint for the index AM. Like in copy_index_constraints(), we make the new catalog entries dependent on the transient relation, so they are dropped along with it automatically. --- src/backend/commands/repack.c | 97 +++++++++++++++++++ .../injection_points/specs/repack.spec | 3 +- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 4d177c868bb..725efa236bc 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -48,6 +48,7 @@ #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" +#include "catalog/pg_attrdef.h" #include "catalog/pg_constraint.h" #include "catalog/pg_inherits.h" #include "catalog/toasting.h" @@ -204,6 +205,7 @@ static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHea static List *build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes); static void copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id); +static void copy_attribute_defaults(Relation old_heap, Relation new_heap); static Relation process_single_relation(RepackStmt *stmt, LOCKMODE lockmode, bool isTopLevel, @@ -1083,6 +1085,13 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose, Assert(CheckRelationOidLockedByMe(OIDNewHeap, AccessExclusiveLock, false)); NewHeap = table_open(OIDNewHeap, NoLock); + /* + * Copy attribute defaults - the executor may need them, in order to + * process the concurrent data changes. In particular, this is related to + * ExecInsertIndexTuples(). + */ + copy_attribute_defaults(OldHeap, NewHeap); + /* Copy the heap data into the new table in the desired order */ copy_table_data(NewHeap, OldHeap, index, snapshot, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); @@ -3434,6 +3443,94 @@ copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id) CommandCounterIncrement(); } +/* + * Create a transient copy of attribute defaults for the transient table. + * + * Like above, the executor needs information on attribute defaults. Once the + * repacking is finished, the catalog entries we create here are dropped. + */ +static void +copy_attribute_defaults(Relation old_heap, Relation new_heap) +{ + Oid old_heap_id = RelationGetRelid(old_heap); + Oid new_heap_id = RelationGetRelid(new_heap); + ScanKeyData skey; + Relation rel; + Relation att_rel = NULL; + TupleDesc desc; + SysScanDesc scan; + HeapTuple tup; + ObjectAddress objrel; + + rel = table_open(AttrDefaultRelationId, RowExclusiveLock); + ObjectAddressSet(objrel, RelationRelationId, new_heap_id); + + ScanKeyInit(&skey, + Anum_pg_attrdef_adrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(old_heap_id)); + scan = systable_beginscan(rel, AttrDefaultIndexId, true, + NULL, 1, &skey); + desc = RelationGetDescr(rel); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_attrdef adform; + Oid oid; + Datum values[Natts_pg_attrdef] = {0}; + bool nulls[Natts_pg_attrdef] = {0}; + bool replaces[Natts_pg_attrdef] = {0}; + HeapTuple new_tup, att_tup, att_new_tup; + ObjectAddress objad; + Datum att_values[Natts_pg_attribute] = {0}; + bool att_nulls[Natts_pg_attribute] = {0}; + bool att_replaces[Natts_pg_attribute] = {0}; + + adform = (Form_pg_attrdef) GETSTRUCT(tup); + Assert(adform->adrelid == old_heap_id); + + oid = GetNewOidWithIndex(rel, AttrDefaultOidIndexId, + Anum_pg_attrdef_oid); + values[Anum_pg_attrdef_oid - 1] = ObjectIdGetDatum(oid); + replaces[Anum_pg_attrdef_oid - 1] = true; + values[Anum_pg_attrdef_adrelid - 1] = ObjectIdGetDatum(new_heap_id); + replaces[Anum_pg_attrdef_adrelid - 1] = true; + + new_tup = heap_modify_tuple(tup, desc, values, nulls, replaces); + + /* Insert it into the catalog. */ + CatalogTupleInsert(rel, new_tup); + + /* Create a dependency so it's removed when we drop the new heap. */ + ObjectAddressSet(objad, AttrDefaultRelationId, oid); + recordDependencyOn(&objad, &objrel, DEPENDENCY_AUTO); + + /* Set atthasdef - new heap has it cleared. */ + att_tup = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(new_heap_id), + ObjectIdGetDatum(adform->adnum)); + if (!HeapTupleIsValid(att_tup)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + adform->adnum, new_heap_id); + + att_values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(true); + att_replaces[Anum_pg_attribute_atthasdef - 1] = true; + + if (att_rel == NULL) + att_rel = table_open(AttributeRelationId, RowExclusiveLock); + att_new_tup = heap_modify_tuple(att_tup, RelationGetDescr(att_rel), + att_values, att_nulls, att_replaces); + ReleaseSysCache(att_tup); + CatalogTupleUpdate(att_rel, &att_new_tup->t_self, att_new_tup); + } + systable_endscan(scan); + + table_close(rel, RowExclusiveLock); + if (att_rel) + table_close(att_rel, RowExclusiveLock); + + CommandCounterIncrement(); +} + /* * Try to start a background worker to perform logical decoding of data * changes applied to relation while REPACK CONCURRENTLY is copying its diff --git a/src/test/modules/injection_points/specs/repack.spec b/src/test/modules/injection_points/specs/repack.spec index d727a9b056b..7896d1456ad 100644 --- a/src/test/modules/injection_points/specs/repack.spec +++ b/src/test/modules/injection_points/specs/repack.spec @@ -3,7 +3,8 @@ setup { CREATE EXTENSION injection_points; - CREATE TABLE repack_test(i int PRIMARY KEY, j int); + CREATE TABLE repack_test(i int PRIMARY KEY, j int, + k int GENERATED ALWAYS AS (j * 2) STORED); INSERT INTO repack_test(i, j) VALUES (1, 1), (2, 2), (3, 3), (4, 4); CREATE TABLE relfilenodes(node oid); -- 2.52.0 --=-=-=-- ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2026-06-22 07:34 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2023-02-15 18:36 [PATCH v16 2/5] refactor code for restoring via shell Nathan Bossart <[email protected]> 2026-06-22 07:34 [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. Antonin Houska <[email protected]> 2026-06-22 07:34 [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. Antonin Houska <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox