public inbox for [email protected]
help / color / mirror / Atom feedFrom: Dmitry Koval <[email protected]>
To: vignesh C <[email protected]>
Cc: Alvaro Herrera <[email protected]>
Cc: PostgreSQL Hackers <[email protected]>
Subject: Re: Operation log for major operations
Date: Sat, 14 Jan 2023 13:17:07 +0300
Message-ID: <[email protected]> (raw)
In-Reply-To: <CALDaNm2tO7sgHy-2Os8sgTWfk=YwwQTcGa-fnJ=Ri69_hLvw8A@mail.gmail.com>
References: <[email protected]>
<[email protected]>
<CALDaNm2tO7sgHy-2Os8sgTWfk=YwwQTcGa-fnJ=Ri69_hLvw8A@mail.gmail.com>
Hi!
>The patch does not apply on top of HEAD ...
Here is a fixed version.
Small additional fixes:
1) added CRC calculation for empty 'pg_control_log' file;
2) added saving 'errno' before calling LWLockRelease and restoring after
that;
3) corrected pg_upgrade for case old cluster does not have
'pg_control_log' file.
--
With best regards,
Dmitry Koval
Postgres Professional: http://postgrespro.com
From 69faac5e9fc11d17d641a0cff3b26806c2930e3f Mon Sep 17 00:00:00 2001
From: Koval Dmitry <[email protected]>
Date: Mon, 14 Nov 2022 21:39:14 +0300
Subject: [PATCH v3] Operation log
---
src/backend/access/transam/xlog.c | 10 +
src/backend/backup/basebackup.c | 1 +
src/backend/storage/lmgr/lwlocknames.txt | 1 +
src/backend/utils/misc/Makefile | 1 +
src/backend/utils/misc/meson.build | 1 +
src/backend/utils/misc/pg_controllog.c | 142 ++++
src/bin/pg_checksums/pg_checksums.c | 1 +
src/bin/pg_resetwal/pg_resetwal.c | 4 +
src/bin/pg_rewind/file_ops.c | 28 +
src/bin/pg_rewind/file_ops.h | 2 +
src/bin/pg_rewind/pg_rewind.c | 59 ++
src/bin/pg_upgrade/controldata.c | 67 ++
src/bin/pg_upgrade/exec.c | 9 +-
src/bin/pg_upgrade/pg_upgrade.c | 2 +
src/bin/pg_upgrade/pg_upgrade.h | 2 +
src/common/Makefile | 1 +
src/common/controllog_utils.c | 683 ++++++++++++++++++
src/common/meson.build | 1 +
src/include/catalog/pg_controllog.h | 142 ++++
src/include/catalog/pg_proc.dat | 9 +
src/include/common/controllog_utils.h | 27 +
src/test/modules/test_misc/meson.build | 1 +
.../modules/test_misc/t/004_operation_log.pl | 136 ++++
src/tools/msvc/Mkvcbuild.pm | 4 +-
24 files changed, 1329 insertions(+), 5 deletions(-)
create mode 100644 src/backend/utils/misc/pg_controllog.c
create mode 100644 src/common/controllog_utils.c
create mode 100644 src/include/catalog/pg_controllog.h
create mode 100644 src/include/common/controllog_utils.h
create mode 100644 src/test/modules/test_misc/t/004_operation_log.pl
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0070d56b0b..6b82acb46b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -68,6 +68,7 @@
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
#include "common/file_utils.h"
#include "executor/instrument.h"
#include "miscadmin.h"
@@ -4774,6 +4775,9 @@ BootStrapXLOG(void)
/* some additional ControlFile fields are set in WriteControlFile() */
WriteControlFile();
+ /* Put information into operation log. */
+ put_operation_log_element(DataDir, OLT_BOOTSTRAP);
+
/* Bootstrap the commit log, too */
BootStrapCLOG();
BootStrapCommitTs();
@@ -5740,8 +5744,14 @@ StartupXLOG(void)
SpinLockRelease(&XLogCtl->info_lck);
UpdateControlFile();
+
LWLockRelease(ControlFileLock);
+ /* Put information into operation log. */
+ if (promoted)
+ put_operation_log_element(DataDir, OLT_PROMOTED);
+ put_operation_log_element(DataDir, OLT_STARTUP);
+
/*
* Shutdown the recovery environment. This must occur after
* RecoverPreparedTransactions() (see notes in lock_twophase_recover())
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
index 3fb9451643..0ca709b5b2 100644
--- a/src/backend/backup/basebackup.c
+++ b/src/backend/backup/basebackup.c
@@ -211,6 +211,7 @@ static const struct exclude_list_item excludeFiles[] =
*/
static const struct exclude_list_item noChecksumFiles[] = {
{"pg_control", false},
+ {"pg_control_log", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index 6c7cf6c295..5673de1669 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -53,3 +53,4 @@ XactTruncationLock 44
# 45 was XactTruncationLock until removal of BackendRandomLock
WrapLimitsVacuumLock 46
NotifyQueueTailLock 47
+ControlLogFileLock 48
diff --git a/src/backend/utils/misc/Makefile b/src/backend/utils/misc/Makefile
index b9ee4eb48a..3fa20e0368 100644
--- a/src/backend/utils/misc/Makefile
+++ b/src/backend/utils/misc/Makefile
@@ -23,6 +23,7 @@ OBJS = \
help_config.o \
pg_config.o \
pg_controldata.o \
+ pg_controllog.o \
pg_rusage.o \
ps_status.o \
queryenvironment.o \
diff --git a/src/backend/utils/misc/meson.build b/src/backend/utils/misc/meson.build
index e3e99ec5cb..9932aa637d 100644
--- a/src/backend/utils/misc/meson.build
+++ b/src/backend/utils/misc/meson.build
@@ -8,6 +8,7 @@ backend_sources += files(
'help_config.c',
'pg_config.c',
'pg_controldata.c',
+ 'pg_controllog.c',
'pg_rusage.c',
'ps_status.c',
'queryenvironment.c',
diff --git a/src/backend/utils/misc/pg_controllog.c b/src/backend/utils/misc/pg_controllog.c
new file mode 100644
index 0000000000..c47c3bf37f
--- /dev/null
+++ b/src/backend/utils/misc/pg_controllog.c
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_controllog.c
+ *
+ * Routines to expose the contents of the control log file via a set of SQL
+ * functions.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/utils/misc/pg_controllog.c
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/htup_details.h"
+#include "access/transam.h"
+#include "access/xlog.h"
+#include "access/xlog_internal.h"
+#include "catalog/pg_type.h"
+#include "common/controllog_utils.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "utils/builtins.h"
+#include "utils/pg_lsn.h"
+#include "utils/timestamp.h"
+
+/*
+ * pg_operation_log
+ *
+ * Returns list of operation log data.
+ * NOTE: this is a set-returning-function (SRF).
+ */
+Datum
+pg_operation_log(PG_FUNCTION_ARGS)
+{
+#define PG_OPERATION_LOG_COLS 6
+ FuncCallContext *funcctx;
+ OperationLogBuffer *log_buffer;
+
+ /*
+ * Initialize tuple descriptor & function call context.
+ */
+ if (SRF_IS_FIRSTCALL())
+ {
+ MemoryContext oldcxt;
+ TupleDesc tupdesc;
+ bool crc_ok;
+
+ /* create a function context for cross-call persistence */
+ funcctx = SRF_FIRSTCALL_INIT();
+
+ /* switch to memory context appropriate for multiple function calls */
+ oldcxt = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ /* read the control file */
+ log_buffer = get_operation_log(DataDir, &crc_ok);
+ if (!crc_ok)
+ ereport(ERROR,
+ (errmsg("calculated CRC checksum does not match value stored in file")));
+
+ tupdesc = CreateTemplateTupleDesc(PG_OPERATION_LOG_COLS);
+
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "event",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "edition",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "version",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 4, "lsn",
+ PG_LSNOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last",
+ TIMESTAMPTZOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 6, "count",
+ INT4OID, -1, 0);
+
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* The only state we need is the operation log buffer. */
+ funcctx->user_fctx = (void *) log_buffer;
+
+ MemoryContextSwitchTo(oldcxt);
+ }
+
+ /* stuff done on every call of the function */
+ funcctx = SRF_PERCALL_SETUP();
+ log_buffer = (OperationLogBuffer *) funcctx->user_fctx;
+
+ if (funcctx->call_cntr < get_operation_log_count(log_buffer))
+ {
+ Datum result;
+ Datum values[PG_OPERATION_LOG_COLS];
+ bool nulls[PG_OPERATION_LOG_COLS];
+ HeapTuple tuple;
+ OperationLogData *data = get_operation_log_element(log_buffer, (uint16) funcctx->call_cntr);
+ int major_version,
+ minor_version,
+ patch_version;
+
+ /*
+ * Form tuple with appropriate data.
+ */
+ MemSet(nulls, 0, sizeof(nulls));
+ MemSet(values, 0, sizeof(values));
+
+ /* event */
+ values[0] = CStringGetTextDatum(get_operation_log_type_name(data->ol_type));
+
+ /* edition */
+ values[1] = CStringGetTextDatum(get_str_edition(data->ol_edition));
+
+ /* version */
+ patch_version = data->ol_version % 100;
+ minor_version = (data->ol_version / 100) % 100;
+ major_version = data->ol_version / 10000;
+ if (major_version < 1000)
+ values[2] = CStringGetTextDatum(psprintf("%u.%u.%u.%u", major_version / 100,
+ major_version % 100,
+ minor_version, patch_version));
+ else
+ values[2] = CStringGetTextDatum(psprintf("%u.%u.%u", major_version / 100,
+ minor_version, patch_version));
+
+ /* lsn */
+ values[3] = LSNGetDatum(data->ol_lsn);
+
+ /* last */
+ values[4] = TimestampTzGetDatum(time_t_to_timestamptz(data->ol_timestamp));
+
+ /* count */
+ values[5] = Int32GetDatum(data->ol_count);
+
+ tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+ result = HeapTupleGetDatum(tuple);
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+
+ /* done when there are no more elements left */
+ SRF_RETURN_DONE(funcctx);
+}
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index aa21007497..32122db023 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -115,6 +115,7 @@ struct exclude_list_item
*/
static const struct exclude_list_item skip[] = {
{"pg_control", false},
+ {"pg_control_log", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index e7ef2b8bd0..5cd3fc29fb 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -50,6 +50,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
#include "common/fe_memutils.h"
#include "common/file_perm.h"
#include "common/logging.h"
@@ -885,6 +886,9 @@ RewriteControlFile(void)
/* The control file gets flushed here. */
update_controlfile(".", &ControlFile, true);
+
+ /* Put information into operation log. */
+ put_operation_log_element(".", OLT_RESETWAL);
}
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 25996b4da4..fb6dc73309 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -354,6 +354,34 @@ slurpFile(const char *datadir, const char *path, size_t *filesize)
return buffer;
}
+/*
+ * Try to open file.
+ * Returns true if file exists.
+ */
+bool
+check_file_exists(const char *datadir, const char *path)
+{
+ char fullpath[MAXPGPATH];
+ int fd;
+
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
+
+ errno = 0;
+ if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
+ {
+ /* File doesn't exist */
+ if (errno == ENOENT)
+ return false;
+
+ pg_fatal("could not open file \"%s\" for reading: %m",
+ fullpath);
+ }
+
+ close(fd);
+
+ return true;
+}
+
/*
* Traverse through all files in a data directory, calling 'callback'
* for each file.
diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h
index 427cf8e0b5..904c697a9c 100644
--- a/src/bin/pg_rewind/file_ops.h
+++ b/src/bin/pg_rewind/file_ops.h
@@ -26,4 +26,6 @@ extern char *slurpFile(const char *datadir, const char *path, size_t *filesize);
typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target);
extern void traverse_datadir(const char *datadir, process_file_callback_t callback);
+extern bool check_file_exists(const char *datadir, const char *path);
+
#endif /* FILE_OPS_H */
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index 858d8d9f2f..eb11531de1 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -19,6 +19,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
#include "common/file_perm.h"
#include "common/restricted_token.h"
#include "common/string.h"
@@ -43,6 +44,8 @@ static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli,
static void digestControlFile(ControlFileData *ControlFile,
const char *content, size_t size);
+static void digestOperationLog(OperationLogBuffer * LogBuffer,
+ const char *content, size_t size);
static void getRestoreCommand(const char *argv0);
static void sanityChecks(void);
static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex);
@@ -53,6 +56,8 @@ static ControlFileData ControlFile_target;
static ControlFileData ControlFile_source;
static ControlFileData ControlFile_source_after;
+static OperationLogBuffer OperationLog_target = {0};
+
const char *progname;
int WalSegSz;
@@ -330,6 +335,15 @@ main(int argc, char **argv)
digestControlFile(&ControlFile_source, buffer, size);
pg_free(buffer);
+ /* Read target operation log for prevent rewriting it. */
+ if (check_file_exists(datadir_target, PG_OPERATION_LOG_FILE))
+ {
+ buffer = slurpFile(datadir_target, PG_OPERATION_LOG_FILE, &size);
+ digestOperationLog(&OperationLog_target, buffer, size);
+ pg_free(buffer);
+ }
+ /* Otherwise we have OperationLog_target with zeros. */
+
sanityChecks();
/*
@@ -672,7 +686,14 @@ perform_rewind(filemap_t *filemap, rewind_source *source,
ControlFile_new.minRecoveryPointTLI = endtli;
ControlFile_new.state = DB_IN_ARCHIVE_RECOVERY;
if (!dry_run)
+ {
update_controlfile(datadir_target, &ControlFile_new, do_sync);
+
+ /* Restore saved operation log. */
+ update_operation_log(datadir_target, &OperationLog_target);
+ /* Put information about "pg_rewind" into operation log. */
+ put_operation_log_element(datadir_target, OLT_REWIND);
+ }
}
static void
@@ -1009,6 +1030,44 @@ digestControlFile(ControlFileData *ControlFile, const char *content,
checkControlFile(ControlFile);
}
+/*
+ * Check CRC of operation log buffer
+ */
+static void
+checkOperationLogBuffer(OperationLogBuffer * LogBuffer)
+{
+ pg_crc32c crc;
+
+ /* Calculate CRC */
+ INIT_CRC32C(crc);
+ COMP_CRC32C(crc,
+ (char *) LogBuffer + sizeof(pg_crc32c),
+ PG_OPERATION_LOG_FULL_SIZE - sizeof(pg_crc32c));
+ FIN_CRC32C(crc);
+
+ /* And simply compare it */
+ if (!EQ_CRC32C(crc, LogBuffer->header.ol_crc))
+ pg_fatal("unexpected operation log CRC");
+}
+
+/*
+ * Verify operation log contents in the buffer 'content', and copy it to
+ * *LogBuffer.
+ */
+static void
+digestOperationLog(OperationLogBuffer * LogBuffer, const char *content,
+ size_t size)
+{
+ if (size != PG_OPERATION_LOG_FULL_SIZE)
+ pg_fatal("unexpected operation log size %d, expected %d",
+ (int) size, PG_OPERATION_LOG_FULL_SIZE);
+
+ memcpy(LogBuffer, content, size);
+
+ /* Additional checks on operation log */
+ checkOperationLogBuffer(LogBuffer);
+}
+
/*
* Get value of GUC parameter restore_command from the target cluster.
*
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 9071a6fd45..df5897c67f 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -14,6 +14,7 @@
#include "pg_upgrade.h"
#include "common/string.h"
+#include "common/controllog_utils.h"
/*
* get_control_data()
@@ -731,3 +732,69 @@ disable_old_cluster(void)
"started once the new cluster has been started.",
old_cluster.pgdata);
}
+
+
+/*
+ * copy_operation_log()
+ *
+ * Copy operation log from the old cluster to the new cluster and put info
+ * about upgrade. If operation log not exists in the old cluster then put
+ * startup message with version info of old cluster.
+ */
+void
+copy_operation_log(void)
+{
+ OperationLogBuffer *log_buffer;
+ bool log_is_empty = true;
+ ClusterInfo *cluster;
+ bool crc_ok;
+ char filename[MAXPGPATH];
+ FILE *fp;
+
+ /* Try to open operation log file in the old cluster. */
+ snprintf(filename, sizeof(filename), "%s/%s", old_cluster.pgdata, PG_OPERATION_LOG_FILE);
+ if ((fp = fopen(filename, "r")) != NULL)
+ {
+ log_is_empty = false;
+ fclose(fp);
+ }
+
+ if (!log_is_empty)
+ {
+ /* Read operation log from the old cluster. */
+ log_buffer = get_operation_log(old_cluster.pgdata, &crc_ok);
+ if (!crc_ok)
+ pg_fatal("pg_control operation log CRC value is incorrect");
+
+ /*
+ * Check operation log records in the old cluster. Need to put
+ * information about old version in case operation log is empty.
+ */
+ log_is_empty = (get_operation_log_count(log_buffer) == 0);
+ }
+ else
+ log_buffer = get_empty_operation_log_buffer();
+
+ if (user_opts.transfer_mode == TRANSFER_MODE_LINK)
+ cluster = &old_cluster;
+ else
+ {
+ cluster = &new_cluster;
+
+ /* Place operation log in the new cluster. */
+ update_operation_log(cluster->pgdata, log_buffer);
+ }
+
+ pfree(log_buffer);
+
+ /* Put information about the old cluster if needed. */
+ if (log_is_empty)
+ put_operation_log_element_version(cluster->pgdata, OLT_STARTUP,
+ ED_PG_ORIGINAL,
+ old_cluster.bin_version_num);
+
+ /*
+ * Put information about upgrade in the operation log of the old cluster.
+ */
+ put_operation_log_element(cluster->pgdata, OLT_UPGRADE);
+}
diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c
index 5b2edebe41..6728a617de 100644
--- a/src/bin/pg_upgrade/exec.c
+++ b/src/bin/pg_upgrade/exec.c
@@ -37,7 +37,8 @@ get_bin_version(ClusterInfo *cluster)
FILE *output;
int rc;
int v1 = 0,
- v2 = 0;
+ v2 = 0,
+ v3 = 0;
snprintf(cmd, sizeof(cmd), "\"%s/pg_ctl\" --version", cluster->bindir);
fflush(NULL);
@@ -52,18 +53,20 @@ get_bin_version(ClusterInfo *cluster)
pg_fatal("could not get pg_ctl version data using %s: %s",
cmd, wait_result_to_str(rc));
- if (sscanf(cmd_output, "%*s %*s %d.%d", &v1, &v2) < 1)
- pg_fatal("could not get pg_ctl version output from %s", cmd);
+ if (sscanf(cmd_output, "%*s %*s %d.%d.%d", &v1, &v2, &v3) < 1)
+ pg_fatal("could not get pg_ctl version output from %s\n", cmd);
if (v1 < 10)
{
/* old style, e.g. 9.6.1 */
cluster->bin_version = v1 * 10000 + v2 * 100;
+ cluster->bin_version_num = (cluster->bin_version + v3) * 100;
}
else
{
/* new style, e.g. 10.1 */
cluster->bin_version = v1 * 10000;
+ cluster->bin_version_num = (cluster->bin_version + v2) * 100;
}
}
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index e5597d3105..df6258ab03 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -194,6 +194,8 @@ main(int argc, char **argv)
check_ok();
}
+ copy_operation_log();
+
create_script_for_old_cluster_deletion(&deletion_script_file_name);
issue_warnings_and_set_wal_level();
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 5f2a116f23..a6dda7b0c3 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -264,6 +264,7 @@ typedef struct
uint32 major_version; /* PG_VERSION of cluster */
char major_version_str[64]; /* string PG_VERSION of cluster */
uint32 bin_version; /* version returned from pg_ctl */
+ uint32 bin_version_num; /* full version (incl. minor part) returned from pg_ctl */
const char *tablespace_suffix; /* directory specification */
} ClusterInfo;
@@ -348,6 +349,7 @@ void create_script_for_old_cluster_deletion(char **deletion_script_file_name);
void get_control_data(ClusterInfo *cluster, bool live_check);
void check_control_data(ControlData *oldctrl, ControlData *newctrl);
void disable_old_cluster(void);
+void copy_operation_log(void);
/* dump.c */
diff --git a/src/common/Makefile b/src/common/Makefile
index 113029bf7b..46cf653466 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -52,6 +52,7 @@ OBJS_COMMON = \
compression.o \
config_info.o \
controldata_utils.o \
+ controllog_utils.o \
d2s.o \
encnames.o \
exec.o \
diff --git a/src/common/controllog_utils.c b/src/common/controllog_utils.c
new file mode 100644
index 0000000000..d309179c33
--- /dev/null
+++ b/src/common/controllog_utils.c
@@ -0,0 +1,683 @@
+/*-------------------------------------------------------------------------
+ *
+ * controllog_utils.c
+ * Common code for operation log file output.
+ *
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/common/controllog_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <time.h>
+
+#include "access/xlog_internal.h"
+#include "catalog/pg_control.h"
+#include "catalog/pg_controllog.h"
+#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
+#include "common/file_perm.h"
+#ifdef FRONTEND
+#include "common/file_utils.h"
+#include "common/logging.h"
+#endif
+#include "port/pg_crc32c.h"
+
+#ifndef FRONTEND
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/lwlock.h"
+#endif
+
+/*
+ * Descriptions of supported operations of operation log.
+ */
+OperationLogTypeDesc OperationLogTypesDescs[] = {
+ {OLT_BOOTSTRAP, OLM_INSERT, "bootstrap"},
+ {OLT_STARTUP, OLM_MERGE, "startup"},
+ {OLT_RESETWAL, OLM_MERGE, "pg_resetwal"},
+ {OLT_REWIND, OLM_MERGE, "pg_rewind"},
+ {OLT_UPGRADE, OLM_INSERT, "pg_upgrade"},
+ {OLT_PROMOTED, OLM_INSERT, "promoted"}
+};
+
+
+/*
+ * calculate_operation_log_crc()
+ *
+ * Calculate CRC of operation log.
+ */
+static pg_crc32c
+calculate_operation_log_crc(OperationLogBuffer * log_buffer)
+{
+ pg_crc32c crc;
+
+ INIT_CRC32C(crc);
+ COMP_CRC32C(crc,
+ (char *) log_buffer + sizeof(pg_crc32c),
+ PG_OPERATION_LOG_FULL_SIZE - sizeof(pg_crc32c));
+ FIN_CRC32C(crc);
+
+ return crc;
+}
+
+/*
+ * get_empty_operation_log()
+ *
+ * Function returns empty operation log buffer.
+ */
+OperationLogBuffer *
+get_empty_operation_log_buffer(void)
+{
+ OperationLogBuffer *log_buffer;
+
+ /* Initialize operation log file with zeros. */
+ log_buffer = palloc0(PG_OPERATION_LOG_FULL_SIZE);
+
+ /* Calculate CRC. */
+ log_buffer->header.ol_crc = calculate_operation_log_crc(log_buffer);
+
+ return log_buffer;
+}
+
+/*
+ * create_operation_log_file()
+ *
+ * Create file for operation log and initialize it with zeros.
+ * Function returns descriptor of created file or -1 in error case.
+ * Function cannot generate report with ERROR and FATAL for correct lock
+ * releasing on top level.
+ */
+static int
+create_operation_log_file(char *ControlLogFilePath)
+{
+ int fd;
+ OperationLogBuffer *log_buffer;
+
+#ifndef FRONTEND
+ fd = OpenTransientFile(ControlLogFilePath, O_RDWR | O_CREAT | PG_BINARY);
+
+ if (fd < 0)
+ {
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not create file \"%s\": %m",
+ PG_OPERATION_LOG_FILE)));
+ return -1;
+ }
+#else
+ fd = open(ControlLogFilePath, O_RDWR | O_CREAT | PG_BINARY, pg_file_create_mode);
+
+ if (fd < 0)
+ pg_fatal("could not create file \"%s\": %m",
+ ControlLogFilePath);
+#endif
+
+ /* Initialize operation log file with zeros. */
+ log_buffer = get_empty_operation_log_buffer();
+
+ errno = 0;
+ if (write(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE) != PG_OPERATION_LOG_FULL_SIZE)
+ {
+ /* If write didn't set errno, assume problem is no disk space. */
+ if (errno == 0)
+ errno = ENOSPC;
+
+#ifndef FRONTEND
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePath);
+#endif
+ pfree(log_buffer);
+ return -1;
+ }
+
+ pfree(log_buffer);
+
+#ifndef FRONTEND
+ if (pg_fsync(fd) != 0)
+ {
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not fsync file \"%s\": %m",
+ ControlLogFilePath)));
+ return -1;
+ }
+#else
+ if (fsync(fd) != 0)
+ pg_fatal("could not fsync file \"%s\": %m", ControlLogFilePath);
+#endif
+
+ if (lseek(fd, 0, SEEK_SET) != 0)
+ {
+#ifndef FRONTEND
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not seek to position 0 of file \"%s\": %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not seek to position 0 of file \"%s\": %m",
+ ControlLogFilePath);
+#endif
+ return -1;
+ }
+
+ return fd;
+}
+
+#define LWLockReleaseSaveErrno(lock) \
+ save_errno = errno; \
+ LWLockRelease(lock); \
+ errno = save_errno; \
+
+/*
+ * get_operation_log()
+ *
+ * Get the operation log ring buffer. The result is returned as a palloc'd copy
+ * of operation log buffer.
+ *
+ * crc_ok_p can be used by the caller to see whether the CRC of the operation
+ * log is correct.
+ */
+OperationLogBuffer *
+get_operation_log(const char *DataDir, bool *crc_ok_p)
+{
+ OperationLogBuffer *log_buffer = NULL;
+ int fd;
+ char ControlLogFilePath[MAXPGPATH];
+ pg_crc32c crc;
+ int r;
+#ifndef FRONTEND
+ int save_errno;
+#endif
+
+ Assert(crc_ok_p);
+
+ snprintf(ControlLogFilePath, MAXPGPATH, "%s/%s", DataDir, PG_OPERATION_LOG_FILE);
+
+#ifndef FRONTEND
+ LWLockAcquire(ControlLogFileLock, LW_EXCLUSIVE);
+ fd = OpenTransientFile(ControlLogFilePath, O_RDONLY | PG_BINARY);
+#else
+ fd = open(ControlLogFilePath, O_RDONLY | PG_BINARY, 0);
+#endif
+ if (fd < 0)
+ {
+ /* File doesn't exist - try to create it. */
+ if (errno == ENOENT)
+ fd = create_operation_log_file(ControlLogFilePath);
+
+ if (fd < 0)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\" for reading: %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not open file \"%s\" for reading: %m",
+ ControlLogFilePath);
+#endif
+ }
+ }
+
+ log_buffer = palloc(PG_OPERATION_LOG_FULL_SIZE);
+
+ r = read(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE);
+ if (r != PG_OPERATION_LOG_FULL_SIZE)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("could not read operation log from the file \"%s\": read %d of %d",
+ ControlLogFilePath, r, PG_OPERATION_LOG_FULL_SIZE)));
+#else
+ pg_fatal("could not read operation log from the file \"%s\": read %d of %d",
+ ControlLogFilePath, r, PG_OPERATION_LOG_FULL_SIZE);
+#endif
+ }
+
+#ifndef FRONTEND
+ if (CloseTransientFile(fd) != 0)
+ {
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not close file \"%s\": %m",
+ ControlLogFilePath)));
+ }
+ LWLockRelease(ControlLogFileLock);
+#else
+ if (close(fd) != 0)
+ pg_fatal("could not close file \"%s\": %m", ControlLogFilePath);
+#endif
+
+ /* Check the CRC. */
+ crc = calculate_operation_log_crc(log_buffer);
+
+ *crc_ok_p = EQ_CRC32C(crc, log_buffer->header.ol_crc);
+
+ return log_buffer;
+}
+
+/*
+ * update_operation_log()
+ *
+ * Update the operation log ring buffer.
+ * Note. To protect against failures a operation log file is written in two
+ * stages: first a temporary file is created, then the temporary file is
+ * renamed to the operation log file.
+ */
+void
+update_operation_log(const char *DataDir, OperationLogBuffer * log_buffer)
+{
+ int fd;
+ char ControlLogFilePath[MAXPGPATH];
+ char ControlLogFilePathTmp[MAXPGPATH];
+#ifndef FRONTEND
+ int save_errno;
+#endif
+
+ snprintf(ControlLogFilePath, sizeof(ControlLogFilePath), "%s/%s", DataDir,
+ PG_OPERATION_LOG_FILE);
+ snprintf(ControlLogFilePathTmp, sizeof(ControlLogFilePathTmp), "%s.tmp",
+ ControlLogFilePath);
+
+ /* Recalculate CRC of operation log. */
+ log_buffer->header.ol_crc = calculate_operation_log_crc(log_buffer);
+
+#ifndef FRONTEND
+ LWLockAcquire(ControlLogFileLock, LW_EXCLUSIVE);
+#endif
+
+ /* Create a temporary file. */
+ fd = create_operation_log_file(ControlLogFilePathTmp);
+ if (fd < 0)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m",
+ ControlLogFilePathTmp)));
+#else
+ pg_fatal("could not open file \"%s\": %m", ControlLogFilePathTmp);
+#endif
+ }
+
+ /* Place operation log buffer into temporary file. */
+ errno = 0;
+ if (write(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE) != PG_OPERATION_LOG_FULL_SIZE)
+ {
+ /* if write didn't set errno, assume problem is no disk space */
+ if (errno == 0)
+ errno = ENOSPC;
+
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePathTmp)));
+#else
+ pg_fatal("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePathTmp);
+#endif
+ }
+
+ /* Close the temporary file. */
+#ifndef FRONTEND
+ if (CloseTransientFile(fd) != 0)
+ {
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not close file \"%s\": %m",
+ ControlLogFilePathTmp)));
+ }
+#else
+ if (close(fd) != 0)
+ pg_fatal("could not close file \"%s\": %m", ControlLogFilePathTmp);
+#endif
+
+ /* Unlink old file with operation log. */
+ if (unlink(ControlLogFilePath) != 0)
+ {
+ /* File can be not exist: ignore this specific error. */
+ if (errno != ENOENT)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not unlink file \"%s\": %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not unlink file \"%s\": %m", ControlLogFilePath);
+#endif
+ }
+ }
+
+ /* Rename temporary file to required name. */
+#ifndef FRONTEND
+ if (durable_rename(ControlLogFilePathTmp, ControlLogFilePath, LOG) != 0)
+ {
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not rename file \"%s\" to \"%s\": %m",
+ ControlLogFilePathTmp, ControlLogFilePath)));
+ }
+ LWLockRelease(ControlLogFileLock);
+#else
+ if (durable_rename(ControlLogFilePathTmp, ControlLogFilePath) != 0)
+ pg_fatal("could not rename file \"%s\" to \"%s\": %m",
+ ControlLogFilePathTmp, ControlLogFilePath);
+#endif
+}
+
+/*
+ * is_enum_value_correct()
+ *
+ * Function returns true in case value is correct value of enum.
+ *
+ * val - test value;
+ * minval - first enum value (correct value);
+ * maxval - last enum value (incorrect value).
+ */
+static bool
+is_enum_value_correct(int16 val, int16 minval, int16 maxval)
+{
+ Assert(val >= minval || val < maxval);
+
+ if (val < minval || val >= maxval)
+ return false;
+ return true;
+}
+
+/*
+ * get_operation_log_type_desc()
+ *
+ * Function returns pointer to OperationLogTypeDesc struct for given type of
+ * operation ol_type.
+ */
+static OperationLogTypeDesc *
+get_operation_log_type_desc(ol_type_enum ol_type)
+{
+ return &OperationLogTypesDescs[ol_type - 1];
+}
+
+/*
+ * fill_operation_log_element()
+ *
+ * Fill new operation log element. Value of ol_lsn is last checkpoint record
+ * pointer.
+ */
+static void
+fill_operation_log_element(ControlFileData *ControlFile,
+ OperationLogTypeDesc * desc,
+ PgNumEdition edition, uint32 version_num,
+ OperationLogData * data)
+{
+ data->ol_type = desc->ol_type;
+ data->ol_edition = edition;
+ data->ol_count = 1;
+ data->ol_version = version_num;
+ data->ol_timestamp = (pg_time_t) time(NULL);
+ data->ol_lsn = ControlFile->checkPoint;
+}
+
+/*
+ * find_operation_log_element_for_merge()
+ *
+ * Find element into operation log ring buffer by ol_type and version.
+ * Returns NULL in case element is not found.
+ */
+static OperationLogData *
+find_operation_log_element_for_merge(ol_type_enum ol_type,
+ OperationLogBuffer * log_buffer,
+ PgNumEdition edition, uint32 version_num)
+{
+ uint32 first = log_buffer->header.ol_first;
+ uint32 count = get_operation_log_count(log_buffer);
+ OperationLogData *data;
+ uint32 i;
+
+ Assert(first < PG_OPERATION_LOG_COUNT && count <= PG_OPERATION_LOG_COUNT);
+
+ for (i = 0; i < count; i++)
+ {
+ data = &log_buffer->data[(first + i) % PG_OPERATION_LOG_COUNT];
+ if (data->ol_type == ol_type &&
+ data->ol_edition == edition &&
+ data->ol_version == version_num)
+ return data;
+ }
+
+ return NULL;
+}
+
+/*
+ * put_operation_log_element(), put_operation_log_element_version()
+ *
+ * Put element into operation log ring buffer.
+ *
+ * DataDir is the path to the top level of the PGDATA directory tree;
+ * ol_type is type of operation;
+ * edition is edition of current PostgreSQL version;
+ * version_num is number of version (for example 13000802 for v13.8.2).
+ *
+ * Note that it is up to the caller to properly lock ControlFileLogLock when
+ * calling this routine in the backend.
+ */
+void
+put_operation_log_element_version(const char *DataDir, ol_type_enum ol_type,
+ PgNumEdition edition, uint32 version_num)
+{
+ OperationLogBuffer *log_buffer;
+ ControlFileData *ControlFile;
+ bool crc_ok;
+ OperationLogTypeDesc *desc;
+
+ if (!is_enum_value_correct(ol_type, OLT_BOOTSTRAP, OLT_NumberOfTypes))
+ {
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("invalid type of operation (%u) for operation log", ol_type)));
+#else
+ pg_fatal("invalid type of operation (%u) for operation log", ol_type);
+#endif
+ }
+
+ desc = get_operation_log_type_desc(ol_type);
+
+ if (!is_enum_value_correct(desc->ol_mode, OLM_MERGE, OLM_NumberOfModes))
+ {
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("invalid mode of operation (%u) for operation log", ol_type)));
+#else
+ pg_fatal("invalid mode of operation (%u) for operation log", ol_type);
+#endif
+ }
+
+ /* get a copy of the control file */
+ ControlFile = get_controlfile(DataDir, &crc_ok);
+ if (!crc_ok)
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("pg_control CRC value is incorrect")));
+#else
+ pg_fatal("pg_control CRC value is incorrect");
+#endif
+
+ /* get a copy of the operation log */
+ log_buffer = get_operation_log(DataDir, &crc_ok);
+ if (!crc_ok)
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("pg_control_log CRC value is incorrect")));
+#else
+ pg_fatal("pg_control_log CRC value is incorrect");
+#endif
+
+ switch (desc->ol_mode)
+ {
+ case OLM_MERGE:
+ {
+ OperationLogData *data;
+
+ data = find_operation_log_element_for_merge(ol_type, log_buffer,
+ edition, version_num);
+ if (data)
+ {
+ /*
+ * We just found the element with the same type and the
+ * same version. Update it.
+ */
+ if (data->ol_count < PG_UINT16_MAX) /* prevent overflow */
+ data->ol_count++;
+ data->ol_timestamp = (pg_time_t) time(NULL);
+ data->ol_lsn = ControlFile->checkPoint;
+ break;
+ }
+ }
+ /* FALLTHROUGH */
+
+ case OLM_INSERT:
+ {
+ uint16 first = log_buffer->header.ol_first;
+ uint16 count = log_buffer->header.ol_count;
+ uint16 current;
+
+ Assert(first < PG_OPERATION_LOG_COUNT && count <= PG_OPERATION_LOG_COUNT);
+
+ if (count == PG_OPERATION_LOG_COUNT)
+ {
+ current = first;
+ /* Owerflow, shift the first element */
+ log_buffer->header.ol_first = (first + 1) % PG_OPERATION_LOG_COUNT;
+ }
+ else
+ {
+ current = first + count;
+ /* Increase number of elements: */
+ log_buffer->header.ol_count++;
+ }
+
+ /* Fill operation log element. */
+ fill_operation_log_element(ControlFile, desc, edition, version_num,
+ &log_buffer->data[current]);
+ break;
+ }
+
+ default:
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("unexpected operation log mode %d",
+ desc->ol_mode)));
+#else
+ pg_fatal("unexpected operation log mode %d", desc->ol_mode);
+#endif
+ }
+
+ update_operation_log(DataDir, log_buffer);
+
+ pfree(log_buffer);
+
+ pfree(ControlFile);
+}
+
+/*
+ * Helper constant for determine current edition.
+ * Here can be custom editions.
+ */
+static const uint8 current_edition = ED_PG_ORIGINAL;
+
+/*
+ * Helper constant for determine current version.
+ * Multiplier 100 used as reserve of last two digits for patch number.
+ */
+static const uint32 current_version_num = PG_VERSION_NUM * 100;
+
+void
+put_operation_log_element(const char *DataDir, ol_type_enum ol_type)
+{
+ put_operation_log_element_version(DataDir, ol_type, current_edition, current_version_num);
+}
+
+/*
+ * get_operation_log_element()
+ *
+ * Returns operation log buffer element with number num.
+ */
+OperationLogData *
+get_operation_log_element(OperationLogBuffer * log_buffer, uint16 num)
+{
+ uint32 first = log_buffer->header.ol_first;
+#ifdef USE_ASSERT_CHECKING
+ uint32 count = get_operation_log_count(log_buffer);
+
+ Assert(num < count);
+#endif
+
+ return &log_buffer->data[(first + num) % PG_OPERATION_LOG_COUNT];
+}
+
+/*
+ * get_operation_log_count()
+ *
+ * Returns number of elements in given operation log buffer.
+ */
+uint16
+get_operation_log_count(OperationLogBuffer * log_buffer)
+{
+ return log_buffer->header.ol_count;
+}
+
+/*
+ * get_operation_log_type_name()
+ *
+ * Returns name of given type.
+ */
+const char *
+get_operation_log_type_name(ol_type_enum ol_type)
+{
+ if (is_enum_value_correct(ol_type, OLT_BOOTSTRAP, OLT_NumberOfTypes))
+ return OperationLogTypesDescs[ol_type - 1].ol_name;
+ else
+ return psprintf("unknown name %u", ol_type);
+}
diff --git a/src/common/meson.build b/src/common/meson.build
index 41bd58ebdf..2516913edd 100644
--- a/src/common/meson.build
+++ b/src/common/meson.build
@@ -6,6 +6,7 @@ common_sources = files(
'checksum_helper.c',
'compression.c',
'controldata_utils.c',
+ 'controllog_utils.c',
'encnames.c',
'exec.c',
'file_perm.c',
diff --git a/src/include/catalog/pg_controllog.h b/src/include/catalog/pg_controllog.h
new file mode 100644
index 0000000000..fddac25fea
--- /dev/null
+++ b/src/include/catalog/pg_controllog.h
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_controllog.h
+ * The system operation log file "pg_control_log" is not a heap
+ * relation.
+ * However, we define it here so that the format is documented.
+ *
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/catalog/pg_controllog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_CONTROLLOG_H
+#define PG_CONTROLLOG_H
+
+#include "access/transam.h"
+#include "access/xlogdefs.h"
+#include "pgtime.h" /* for pg_time_t */
+#include "port/pg_crc32c.h"
+
+#define PG_OPERATION_LOG_FILE "global/pg_control_log"
+
+/*
+ * Type of operation for operation log.
+ */
+typedef enum
+{
+ OLT_BOOTSTRAP = 1, /* bootstrap */
+ OLT_STARTUP, /* server startup */
+ OLT_RESETWAL, /* pg_resetwal */
+ OLT_REWIND, /* pg_rewind */
+ OLT_UPGRADE, /* pg_upgrade */
+ OLT_PROMOTED, /* promoted */
+ OLT_NumberOfTypes /* should be last */
+} ol_type_enum;
+
+/*
+ * Mode of operation processing.
+ */
+typedef enum
+{
+ OLM_MERGE = 1, /* insert element only if not exists element
+ * with the same ol_type and ol_version;
+ * otherwise update existing element */
+ OLM_INSERT, /* insert element into ring buffer 'as is' */
+ OLM_NumberOfModes /* should be last */
+} ol_mode_enum;
+
+/*
+ * Helper struct for describing supported operations.
+ */
+typedef struct OperationLogTypeDesc
+{
+ ol_type_enum ol_type; /* element type */
+ ol_mode_enum ol_mode; /* element mode */
+ const char *ol_name; /* display name of element */
+} OperationLogTypeDesc;
+
+/*
+ * Element of operation log ring buffer (24 bytes).
+ */
+typedef struct OperationLogData
+{
+ uint8 ol_type; /* operation type */
+ uint8 ol_edition; /* postgres edition */
+ uint16 ol_count; /* number of operations */
+ uint32 ol_version; /* postgres version */
+ pg_time_t ol_timestamp; /* = int64, operation date/time */
+ XLogRecPtr ol_lsn; /* = uint64, last check point record ptr */
+} OperationLogData;
+
+/*
+ * Header of operation log ring buffer (8 bytes).
+ */
+typedef struct OperationLogHeader
+{
+ pg_crc32c ol_crc; /* CRC of operation log ... MUST BE FIRST! */
+ uint16 ol_first; /* position of first ring buffer element */
+ uint16 ol_count; /* number of elements in ring buffer */
+} OperationLogHeader;
+
+/*
+ * Whole size of the operation log ring buffer (with header).
+ */
+#define PG_OPERATION_LOG_FULL_SIZE 8192
+
+/*
+ * Size of elements of the operation log ring buffer.
+ * Value must be a multiple of sizeof(OperationLogData).
+ */
+#define PG_OPERATION_LOG_SIZE (PG_OPERATION_LOG_FULL_SIZE - sizeof(OperationLogHeader))
+
+/*
+ * Number of elements in the operation log.
+ */
+#define PG_OPERATION_LOG_COUNT (PG_OPERATION_LOG_SIZE / sizeof(OperationLogData))
+
+/*
+ * Operation log ring buffer.
+ */
+typedef struct OperationLogBuffer
+{
+ OperationLogHeader header;
+ OperationLogData data[PG_OPERATION_LOG_COUNT];
+
+} OperationLogBuffer;
+
+StaticAssertDecl(sizeof(OperationLogBuffer) == PG_OPERATION_LOG_FULL_SIZE,
+ "structure OperationLogBuffer must have size PG_OPERATION_LOG_FULL_SIZE");
+
+/* Enum for postgres edition. */
+typedef enum
+{
+ ED_PG_ORIGINAL = 0
+ /* Here can be custom editions */
+} PgNumEdition;
+
+#define ED_PG_ORIGINAL_STR "vanilla"
+#define ED_UNKNOWN_STR "unknown"
+
+/*
+ * get_str_edition()
+ *
+ * Returns edition string by edition number.
+ */
+static inline const char *
+get_str_edition(PgNumEdition edition)
+{
+ switch (edition)
+ {
+ case ED_PG_ORIGINAL:
+ return ED_PG_ORIGINAL_STR;
+
+ /* Here can be custom editions */
+ }
+ return ED_UNKNOWN_STR;
+}
+
+#endif /* PG_CONTROLLOG_H */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 3810de7b22..37397f2c86 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11891,4 +11891,13 @@
prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary',
prosrc => 'brin_minmax_multi_summary_send' },
+# operation log function
+{ oid => '8110', descr => 'show operation log',
+ proname => 'pg_operation_log', prorows => '170', proretset => 't',
+ provolatile => 'v', prorettype => 'record', proargtypes => '',
+ proallargtypes => '{text,text,text,pg_lsn,timestamptz,int4}',
+ proargmodes => '{o,o,o,o,o,o}',
+ proargnames => '{event,edition,version,lsn,last,count}',
+ prosrc => 'pg_operation_log' },
+
]
diff --git a/src/include/common/controllog_utils.h b/src/include/common/controllog_utils.h
new file mode 100644
index 0000000000..dc5c01e87f
--- /dev/null
+++ b/src/include/common/controllog_utils.h
@@ -0,0 +1,27 @@
+/*
+ * controllog_utils.h
+ * Common code for pg_control_log output
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/controllog_utils.h
+ */
+#ifndef COMMON_CONTROLLOG_UTILS_H
+#define COMMON_CONTROLLOG_UTILS_H
+
+#include "catalog/pg_controllog.h"
+
+extern OperationLogBuffer * get_operation_log(const char *DataDir, bool *crc_ok_p);
+extern OperationLogBuffer * get_empty_operation_log_buffer(void);
+extern void update_operation_log(const char *DataDir, OperationLogBuffer * log_buffer);
+
+extern void put_operation_log_element(const char *DataDir, ol_type_enum ol_type);
+extern void put_operation_log_element_version(const char *DataDir, ol_type_enum ol_type,
+ PgNumEdition edition, uint32 version_num);
+extern uint16 get_operation_log_count(OperationLogBuffer * log_buffer);
+extern OperationLogData * get_operation_log_element(OperationLogBuffer * log_buffer,
+ uint16 num);
+extern const char *get_operation_log_type_name(ol_type_enum ol_type);
+
+#endif /* COMMON_CONTROLLOG_UTILS_H */
diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build
index 21bde427b4..ec698284af 100644
--- a/src/test/modules/test_misc/meson.build
+++ b/src/test/modules/test_misc/meson.build
@@ -9,6 +9,7 @@ tests += {
't/001_constraint_validation.pl',
't/002_tablespace.pl',
't/003_check_guc.pl',
+ 't/004_operation_log.pl',
],
},
}
diff --git a/src/test/modules/test_misc/t/004_operation_log.pl b/src/test/modules/test_misc/t/004_operation_log.pl
new file mode 100644
index 0000000000..3a07f78ffa
--- /dev/null
+++ b/src/test/modules/test_misc/t/004_operation_log.pl
@@ -0,0 +1,136 @@
+
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test for operation log.
+#
+# Some events like
+# "bootstrap", "startup", "pg_rewind", "pg_resetwal", "promoted", "pg_upgrade"
+# should be registered in operation log.
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+use File::Copy;
+
+# Create and start primary node
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(allows_streaming => 1);
+$node_primary->append_conf('postgresql.conf', qq(wal_keep_size = 100MB));
+$node_primary->start;
+
+# Get server version
+my $server_version = $node_primary->safe_psql("postgres", "SELECT current_setting('server_version_num');") + 0;
+my $major_version = $server_version / 10000;
+my $minor_version = $server_version % 100;
+
+# Get primary node backup
+$node_primary->backup('primary_backup');
+
+# Initialize standby node from backup
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, 'primary_backup', has_streaming => 1);
+$node_standby->start;
+
+# Promote the standby
+$node_standby->promote;
+
+# Stop standby node
+$node_standby->stop;
+
+my $node_standby_pgdata = $node_standby->data_dir;
+my $node_primary_connstr = $node_primary->connstr;
+
+# Keep a temporary postgresql.conf or it would be overwritten during the rewind.
+my $tmp_folder = PostgreSQL::Test::Utils::tempdir;
+copy("$node_standby_pgdata/postgresql.conf", "$tmp_folder/node_standby-postgresql.conf.tmp");
+
+# Get "pg_rewind" event
+command_ok(
+ [
+ 'pg_rewind',
+ "--source-server=$node_primary_connstr",
+ "--target-pgdata=$node_standby_pgdata",
+ "--debug"
+ ],
+ 'run pg_rewind');
+
+# Move back postgresql.conf with old settings
+move("$tmp_folder/node_standby-postgresql.conf.tmp", "$node_standby_pgdata/postgresql.conf");
+
+# Start and stop standby before resetwal and upgrade
+$node_standby->start;
+$node_standby->stop;
+
+# Get first "pg_resetwal" event
+system_or_bail('pg_resetwal', '-f', $node_standby->data_dir);
+
+# Get second "pg_resetwal" event
+system_or_bail('pg_resetwal', '-f', $node_standby->data_dir);
+
+# Initialize a new node for the upgrade
+my $node_new = PostgreSQL::Test::Cluster->new('new');
+$node_new->init;
+
+my $bindir_new = $node_new->config_data('--bindir');
+my $bindir_standby = $node_standby->config_data('--bindir');
+
+# We want to run pg_upgrade in the build directory so that any files generated
+# finish in it, like delete_old_cluster.{sh,bat}.
+chdir ${PostgreSQL::Test::Utils::tmp_check};
+
+# Run pg_upgrade
+command_ok(
+ [
+ 'pg_upgrade', '--no-sync', '-d', $node_standby->data_dir,
+ '-D', $node_new->data_dir, '-b', $bindir_standby,
+ '-B', $bindir_new, '-s', $node_new->host,
+ '-p', $node_standby->port, '-P', $node_new->port
+ ],
+ 'run pg_upgrade');
+#
+# Need to check operation log
+#
+sub check_event
+{
+ my $event_name = shift;
+ my $result = shift;
+ my $func_args = shift ? "sum(count), count(*)" : "count(*)";
+
+ my $psql_stdout = $node_new->safe_psql('postgres', qq(
+ SELECT
+ $func_args,
+ min(split_part(version,'.','1')),
+ min(split_part(version,'.','2'))
+ FROM
+ pg_operation_log()
+ WHERE
+ event='$event_name'));
+
+ is($psql_stdout, $result, 'check number of event ' . $event_name);
+ return;
+}
+#Start new node
+$node_new->start;
+
+# Check number of event "bootstrap"
+check_event('bootstrap', qq(1|1|$major_version|$minor_version), 1);
+
+# Check number of event "startup"
+check_event('startup', qq(1|$major_version|$minor_version), 0);
+
+# Check number of event "promoted"
+check_event('promoted', qq(1|1|$major_version|$minor_version), 1);
+
+# Check number of event "pg_upgrade"
+check_event('pg_upgrade', qq(1|1|$major_version|$minor_version), 1);
+
+# Check number of event "pg_resetwal"
+check_event('pg_resetwal', qq(2|1|$major_version|$minor_version), 1);
+
+# Check number of event "pg_rewind"
+check_event('pg_rewind', qq(1|1|$major_version|$minor_version), 1);
+
+done_testing();
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index f1c9ddf4a0..0cfd96d9ba 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -134,8 +134,8 @@ sub mkvcbuild
our @pgcommonallfiles = qw(
archive.c base64.c checksum_helper.c compression.c
- config_info.c controldata_utils.c d2s.c encnames.c exec.c
- f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
+ config_info.c controldata_utils.c controllog_utils.c d2s.c encnames.c
+ exec.c f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
keywords.c kwlookup.c link-canary.c md5_common.c percentrepl.c
pg_get_line.c pg_lzcompress.c pg_prng.c pgfnames.c psprintf.c relpath.c
rmtree.c saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c
--
2.31.0.windows.1
Attachments:
[text/plain] v3-0001-Operation-log.patch (48.8K, ../[email protected]/2-v3-0001-Operation-log.patch)
download | inline diff:
From 69faac5e9fc11d17d641a0cff3b26806c2930e3f Mon Sep 17 00:00:00 2001
From: Koval Dmitry <[email protected]>
Date: Mon, 14 Nov 2022 21:39:14 +0300
Subject: [PATCH v3] Operation log
---
src/backend/access/transam/xlog.c | 10 +
src/backend/backup/basebackup.c | 1 +
src/backend/storage/lmgr/lwlocknames.txt | 1 +
src/backend/utils/misc/Makefile | 1 +
src/backend/utils/misc/meson.build | 1 +
src/backend/utils/misc/pg_controllog.c | 142 ++++
src/bin/pg_checksums/pg_checksums.c | 1 +
src/bin/pg_resetwal/pg_resetwal.c | 4 +
src/bin/pg_rewind/file_ops.c | 28 +
src/bin/pg_rewind/file_ops.h | 2 +
src/bin/pg_rewind/pg_rewind.c | 59 ++
src/bin/pg_upgrade/controldata.c | 67 ++
src/bin/pg_upgrade/exec.c | 9 +-
src/bin/pg_upgrade/pg_upgrade.c | 2 +
src/bin/pg_upgrade/pg_upgrade.h | 2 +
src/common/Makefile | 1 +
src/common/controllog_utils.c | 683 ++++++++++++++++++
src/common/meson.build | 1 +
src/include/catalog/pg_controllog.h | 142 ++++
src/include/catalog/pg_proc.dat | 9 +
src/include/common/controllog_utils.h | 27 +
src/test/modules/test_misc/meson.build | 1 +
.../modules/test_misc/t/004_operation_log.pl | 136 ++++
src/tools/msvc/Mkvcbuild.pm | 4 +-
24 files changed, 1329 insertions(+), 5 deletions(-)
create mode 100644 src/backend/utils/misc/pg_controllog.c
create mode 100644 src/common/controllog_utils.c
create mode 100644 src/include/catalog/pg_controllog.h
create mode 100644 src/include/common/controllog_utils.h
create mode 100644 src/test/modules/test_misc/t/004_operation_log.pl
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0070d56b0b..6b82acb46b 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -68,6 +68,7 @@
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
#include "common/file_utils.h"
#include "executor/instrument.h"
#include "miscadmin.h"
@@ -4774,6 +4775,9 @@ BootStrapXLOG(void)
/* some additional ControlFile fields are set in WriteControlFile() */
WriteControlFile();
+ /* Put information into operation log. */
+ put_operation_log_element(DataDir, OLT_BOOTSTRAP);
+
/* Bootstrap the commit log, too */
BootStrapCLOG();
BootStrapCommitTs();
@@ -5740,8 +5744,14 @@ StartupXLOG(void)
SpinLockRelease(&XLogCtl->info_lck);
UpdateControlFile();
+
LWLockRelease(ControlFileLock);
+ /* Put information into operation log. */
+ if (promoted)
+ put_operation_log_element(DataDir, OLT_PROMOTED);
+ put_operation_log_element(DataDir, OLT_STARTUP);
+
/*
* Shutdown the recovery environment. This must occur after
* RecoverPreparedTransactions() (see notes in lock_twophase_recover())
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
index 3fb9451643..0ca709b5b2 100644
--- a/src/backend/backup/basebackup.c
+++ b/src/backend/backup/basebackup.c
@@ -211,6 +211,7 @@ static const struct exclude_list_item excludeFiles[] =
*/
static const struct exclude_list_item noChecksumFiles[] = {
{"pg_control", false},
+ {"pg_control_log", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index 6c7cf6c295..5673de1669 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -53,3 +53,4 @@ XactTruncationLock 44
# 45 was XactTruncationLock until removal of BackendRandomLock
WrapLimitsVacuumLock 46
NotifyQueueTailLock 47
+ControlLogFileLock 48
diff --git a/src/backend/utils/misc/Makefile b/src/backend/utils/misc/Makefile
index b9ee4eb48a..3fa20e0368 100644
--- a/src/backend/utils/misc/Makefile
+++ b/src/backend/utils/misc/Makefile
@@ -23,6 +23,7 @@ OBJS = \
help_config.o \
pg_config.o \
pg_controldata.o \
+ pg_controllog.o \
pg_rusage.o \
ps_status.o \
queryenvironment.o \
diff --git a/src/backend/utils/misc/meson.build b/src/backend/utils/misc/meson.build
index e3e99ec5cb..9932aa637d 100644
--- a/src/backend/utils/misc/meson.build
+++ b/src/backend/utils/misc/meson.build
@@ -8,6 +8,7 @@ backend_sources += files(
'help_config.c',
'pg_config.c',
'pg_controldata.c',
+ 'pg_controllog.c',
'pg_rusage.c',
'ps_status.c',
'queryenvironment.c',
diff --git a/src/backend/utils/misc/pg_controllog.c b/src/backend/utils/misc/pg_controllog.c
new file mode 100644
index 0000000000..c47c3bf37f
--- /dev/null
+++ b/src/backend/utils/misc/pg_controllog.c
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_controllog.c
+ *
+ * Routines to expose the contents of the control log file via a set of SQL
+ * functions.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/utils/misc/pg_controllog.c
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/htup_details.h"
+#include "access/transam.h"
+#include "access/xlog.h"
+#include "access/xlog_internal.h"
+#include "catalog/pg_type.h"
+#include "common/controllog_utils.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "utils/builtins.h"
+#include "utils/pg_lsn.h"
+#include "utils/timestamp.h"
+
+/*
+ * pg_operation_log
+ *
+ * Returns list of operation log data.
+ * NOTE: this is a set-returning-function (SRF).
+ */
+Datum
+pg_operation_log(PG_FUNCTION_ARGS)
+{
+#define PG_OPERATION_LOG_COLS 6
+ FuncCallContext *funcctx;
+ OperationLogBuffer *log_buffer;
+
+ /*
+ * Initialize tuple descriptor & function call context.
+ */
+ if (SRF_IS_FIRSTCALL())
+ {
+ MemoryContext oldcxt;
+ TupleDesc tupdesc;
+ bool crc_ok;
+
+ /* create a function context for cross-call persistence */
+ funcctx = SRF_FIRSTCALL_INIT();
+
+ /* switch to memory context appropriate for multiple function calls */
+ oldcxt = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ /* read the control file */
+ log_buffer = get_operation_log(DataDir, &crc_ok);
+ if (!crc_ok)
+ ereport(ERROR,
+ (errmsg("calculated CRC checksum does not match value stored in file")));
+
+ tupdesc = CreateTemplateTupleDesc(PG_OPERATION_LOG_COLS);
+
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "event",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "edition",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "version",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 4, "lsn",
+ PG_LSNOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 5, "last",
+ TIMESTAMPTZOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 6, "count",
+ INT4OID, -1, 0);
+
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* The only state we need is the operation log buffer. */
+ funcctx->user_fctx = (void *) log_buffer;
+
+ MemoryContextSwitchTo(oldcxt);
+ }
+
+ /* stuff done on every call of the function */
+ funcctx = SRF_PERCALL_SETUP();
+ log_buffer = (OperationLogBuffer *) funcctx->user_fctx;
+
+ if (funcctx->call_cntr < get_operation_log_count(log_buffer))
+ {
+ Datum result;
+ Datum values[PG_OPERATION_LOG_COLS];
+ bool nulls[PG_OPERATION_LOG_COLS];
+ HeapTuple tuple;
+ OperationLogData *data = get_operation_log_element(log_buffer, (uint16) funcctx->call_cntr);
+ int major_version,
+ minor_version,
+ patch_version;
+
+ /*
+ * Form tuple with appropriate data.
+ */
+ MemSet(nulls, 0, sizeof(nulls));
+ MemSet(values, 0, sizeof(values));
+
+ /* event */
+ values[0] = CStringGetTextDatum(get_operation_log_type_name(data->ol_type));
+
+ /* edition */
+ values[1] = CStringGetTextDatum(get_str_edition(data->ol_edition));
+
+ /* version */
+ patch_version = data->ol_version % 100;
+ minor_version = (data->ol_version / 100) % 100;
+ major_version = data->ol_version / 10000;
+ if (major_version < 1000)
+ values[2] = CStringGetTextDatum(psprintf("%u.%u.%u.%u", major_version / 100,
+ major_version % 100,
+ minor_version, patch_version));
+ else
+ values[2] = CStringGetTextDatum(psprintf("%u.%u.%u", major_version / 100,
+ minor_version, patch_version));
+
+ /* lsn */
+ values[3] = LSNGetDatum(data->ol_lsn);
+
+ /* last */
+ values[4] = TimestampTzGetDatum(time_t_to_timestamptz(data->ol_timestamp));
+
+ /* count */
+ values[5] = Int32GetDatum(data->ol_count);
+
+ tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+ result = HeapTupleGetDatum(tuple);
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+
+ /* done when there are no more elements left */
+ SRF_RETURN_DONE(funcctx);
+}
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index aa21007497..32122db023 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -115,6 +115,7 @@ struct exclude_list_item
*/
static const struct exclude_list_item skip[] = {
{"pg_control", false},
+ {"pg_control_log", false},
{"pg_filenode.map", false},
{"pg_internal.init", true},
{"PG_VERSION", false},
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index e7ef2b8bd0..5cd3fc29fb 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -50,6 +50,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
#include "common/fe_memutils.h"
#include "common/file_perm.h"
#include "common/logging.h"
@@ -885,6 +886,9 @@ RewriteControlFile(void)
/* The control file gets flushed here. */
update_controlfile(".", &ControlFile, true);
+
+ /* Put information into operation log. */
+ put_operation_log_element(".", OLT_RESETWAL);
}
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 25996b4da4..fb6dc73309 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -354,6 +354,34 @@ slurpFile(const char *datadir, const char *path, size_t *filesize)
return buffer;
}
+/*
+ * Try to open file.
+ * Returns true if file exists.
+ */
+bool
+check_file_exists(const char *datadir, const char *path)
+{
+ char fullpath[MAXPGPATH];
+ int fd;
+
+ snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
+
+ errno = 0;
+ if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
+ {
+ /* File doesn't exist */
+ if (errno == ENOENT)
+ return false;
+
+ pg_fatal("could not open file \"%s\" for reading: %m",
+ fullpath);
+ }
+
+ close(fd);
+
+ return true;
+}
+
/*
* Traverse through all files in a data directory, calling 'callback'
* for each file.
diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h
index 427cf8e0b5..904c697a9c 100644
--- a/src/bin/pg_rewind/file_ops.h
+++ b/src/bin/pg_rewind/file_ops.h
@@ -26,4 +26,6 @@ extern char *slurpFile(const char *datadir, const char *path, size_t *filesize);
typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target);
extern void traverse_datadir(const char *datadir, process_file_callback_t callback);
+extern bool check_file_exists(const char *datadir, const char *path);
+
#endif /* FILE_OPS_H */
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index 858d8d9f2f..eb11531de1 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -19,6 +19,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
#include "common/file_perm.h"
#include "common/restricted_token.h"
#include "common/string.h"
@@ -43,6 +44,8 @@ static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli,
static void digestControlFile(ControlFileData *ControlFile,
const char *content, size_t size);
+static void digestOperationLog(OperationLogBuffer * LogBuffer,
+ const char *content, size_t size);
static void getRestoreCommand(const char *argv0);
static void sanityChecks(void);
static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex);
@@ -53,6 +56,8 @@ static ControlFileData ControlFile_target;
static ControlFileData ControlFile_source;
static ControlFileData ControlFile_source_after;
+static OperationLogBuffer OperationLog_target = {0};
+
const char *progname;
int WalSegSz;
@@ -330,6 +335,15 @@ main(int argc, char **argv)
digestControlFile(&ControlFile_source, buffer, size);
pg_free(buffer);
+ /* Read target operation log for prevent rewriting it. */
+ if (check_file_exists(datadir_target, PG_OPERATION_LOG_FILE))
+ {
+ buffer = slurpFile(datadir_target, PG_OPERATION_LOG_FILE, &size);
+ digestOperationLog(&OperationLog_target, buffer, size);
+ pg_free(buffer);
+ }
+ /* Otherwise we have OperationLog_target with zeros. */
+
sanityChecks();
/*
@@ -672,7 +686,14 @@ perform_rewind(filemap_t *filemap, rewind_source *source,
ControlFile_new.minRecoveryPointTLI = endtli;
ControlFile_new.state = DB_IN_ARCHIVE_RECOVERY;
if (!dry_run)
+ {
update_controlfile(datadir_target, &ControlFile_new, do_sync);
+
+ /* Restore saved operation log. */
+ update_operation_log(datadir_target, &OperationLog_target);
+ /* Put information about "pg_rewind" into operation log. */
+ put_operation_log_element(datadir_target, OLT_REWIND);
+ }
}
static void
@@ -1009,6 +1030,44 @@ digestControlFile(ControlFileData *ControlFile, const char *content,
checkControlFile(ControlFile);
}
+/*
+ * Check CRC of operation log buffer
+ */
+static void
+checkOperationLogBuffer(OperationLogBuffer * LogBuffer)
+{
+ pg_crc32c crc;
+
+ /* Calculate CRC */
+ INIT_CRC32C(crc);
+ COMP_CRC32C(crc,
+ (char *) LogBuffer + sizeof(pg_crc32c),
+ PG_OPERATION_LOG_FULL_SIZE - sizeof(pg_crc32c));
+ FIN_CRC32C(crc);
+
+ /* And simply compare it */
+ if (!EQ_CRC32C(crc, LogBuffer->header.ol_crc))
+ pg_fatal("unexpected operation log CRC");
+}
+
+/*
+ * Verify operation log contents in the buffer 'content', and copy it to
+ * *LogBuffer.
+ */
+static void
+digestOperationLog(OperationLogBuffer * LogBuffer, const char *content,
+ size_t size)
+{
+ if (size != PG_OPERATION_LOG_FULL_SIZE)
+ pg_fatal("unexpected operation log size %d, expected %d",
+ (int) size, PG_OPERATION_LOG_FULL_SIZE);
+
+ memcpy(LogBuffer, content, size);
+
+ /* Additional checks on operation log */
+ checkOperationLogBuffer(LogBuffer);
+}
+
/*
* Get value of GUC parameter restore_command from the target cluster.
*
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 9071a6fd45..df5897c67f 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -14,6 +14,7 @@
#include "pg_upgrade.h"
#include "common/string.h"
+#include "common/controllog_utils.h"
/*
* get_control_data()
@@ -731,3 +732,69 @@ disable_old_cluster(void)
"started once the new cluster has been started.",
old_cluster.pgdata);
}
+
+
+/*
+ * copy_operation_log()
+ *
+ * Copy operation log from the old cluster to the new cluster and put info
+ * about upgrade. If operation log not exists in the old cluster then put
+ * startup message with version info of old cluster.
+ */
+void
+copy_operation_log(void)
+{
+ OperationLogBuffer *log_buffer;
+ bool log_is_empty = true;
+ ClusterInfo *cluster;
+ bool crc_ok;
+ char filename[MAXPGPATH];
+ FILE *fp;
+
+ /* Try to open operation log file in the old cluster. */
+ snprintf(filename, sizeof(filename), "%s/%s", old_cluster.pgdata, PG_OPERATION_LOG_FILE);
+ if ((fp = fopen(filename, "r")) != NULL)
+ {
+ log_is_empty = false;
+ fclose(fp);
+ }
+
+ if (!log_is_empty)
+ {
+ /* Read operation log from the old cluster. */
+ log_buffer = get_operation_log(old_cluster.pgdata, &crc_ok);
+ if (!crc_ok)
+ pg_fatal("pg_control operation log CRC value is incorrect");
+
+ /*
+ * Check operation log records in the old cluster. Need to put
+ * information about old version in case operation log is empty.
+ */
+ log_is_empty = (get_operation_log_count(log_buffer) == 0);
+ }
+ else
+ log_buffer = get_empty_operation_log_buffer();
+
+ if (user_opts.transfer_mode == TRANSFER_MODE_LINK)
+ cluster = &old_cluster;
+ else
+ {
+ cluster = &new_cluster;
+
+ /* Place operation log in the new cluster. */
+ update_operation_log(cluster->pgdata, log_buffer);
+ }
+
+ pfree(log_buffer);
+
+ /* Put information about the old cluster if needed. */
+ if (log_is_empty)
+ put_operation_log_element_version(cluster->pgdata, OLT_STARTUP,
+ ED_PG_ORIGINAL,
+ old_cluster.bin_version_num);
+
+ /*
+ * Put information about upgrade in the operation log of the old cluster.
+ */
+ put_operation_log_element(cluster->pgdata, OLT_UPGRADE);
+}
diff --git a/src/bin/pg_upgrade/exec.c b/src/bin/pg_upgrade/exec.c
index 5b2edebe41..6728a617de 100644
--- a/src/bin/pg_upgrade/exec.c
+++ b/src/bin/pg_upgrade/exec.c
@@ -37,7 +37,8 @@ get_bin_version(ClusterInfo *cluster)
FILE *output;
int rc;
int v1 = 0,
- v2 = 0;
+ v2 = 0,
+ v3 = 0;
snprintf(cmd, sizeof(cmd), "\"%s/pg_ctl\" --version", cluster->bindir);
fflush(NULL);
@@ -52,18 +53,20 @@ get_bin_version(ClusterInfo *cluster)
pg_fatal("could not get pg_ctl version data using %s: %s",
cmd, wait_result_to_str(rc));
- if (sscanf(cmd_output, "%*s %*s %d.%d", &v1, &v2) < 1)
- pg_fatal("could not get pg_ctl version output from %s", cmd);
+ if (sscanf(cmd_output, "%*s %*s %d.%d.%d", &v1, &v2, &v3) < 1)
+ pg_fatal("could not get pg_ctl version output from %s\n", cmd);
if (v1 < 10)
{
/* old style, e.g. 9.6.1 */
cluster->bin_version = v1 * 10000 + v2 * 100;
+ cluster->bin_version_num = (cluster->bin_version + v3) * 100;
}
else
{
/* new style, e.g. 10.1 */
cluster->bin_version = v1 * 10000;
+ cluster->bin_version_num = (cluster->bin_version + v2) * 100;
}
}
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index e5597d3105..df6258ab03 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -194,6 +194,8 @@ main(int argc, char **argv)
check_ok();
}
+ copy_operation_log();
+
create_script_for_old_cluster_deletion(&deletion_script_file_name);
issue_warnings_and_set_wal_level();
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 5f2a116f23..a6dda7b0c3 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -264,6 +264,7 @@ typedef struct
uint32 major_version; /* PG_VERSION of cluster */
char major_version_str[64]; /* string PG_VERSION of cluster */
uint32 bin_version; /* version returned from pg_ctl */
+ uint32 bin_version_num; /* full version (incl. minor part) returned from pg_ctl */
const char *tablespace_suffix; /* directory specification */
} ClusterInfo;
@@ -348,6 +349,7 @@ void create_script_for_old_cluster_deletion(char **deletion_script_file_name);
void get_control_data(ClusterInfo *cluster, bool live_check);
void check_control_data(ControlData *oldctrl, ControlData *newctrl);
void disable_old_cluster(void);
+void copy_operation_log(void);
/* dump.c */
diff --git a/src/common/Makefile b/src/common/Makefile
index 113029bf7b..46cf653466 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -52,6 +52,7 @@ OBJS_COMMON = \
compression.o \
config_info.o \
controldata_utils.o \
+ controllog_utils.o \
d2s.o \
encnames.o \
exec.o \
diff --git a/src/common/controllog_utils.c b/src/common/controllog_utils.c
new file mode 100644
index 0000000000..d309179c33
--- /dev/null
+++ b/src/common/controllog_utils.c
@@ -0,0 +1,683 @@
+/*-------------------------------------------------------------------------
+ *
+ * controllog_utils.c
+ * Common code for operation log file output.
+ *
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/common/controllog_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <time.h>
+
+#include "access/xlog_internal.h"
+#include "catalog/pg_control.h"
+#include "catalog/pg_controllog.h"
+#include "common/controldata_utils.h"
+#include "common/controllog_utils.h"
+#include "common/file_perm.h"
+#ifdef FRONTEND
+#include "common/file_utils.h"
+#include "common/logging.h"
+#endif
+#include "port/pg_crc32c.h"
+
+#ifndef FRONTEND
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/lwlock.h"
+#endif
+
+/*
+ * Descriptions of supported operations of operation log.
+ */
+OperationLogTypeDesc OperationLogTypesDescs[] = {
+ {OLT_BOOTSTRAP, OLM_INSERT, "bootstrap"},
+ {OLT_STARTUP, OLM_MERGE, "startup"},
+ {OLT_RESETWAL, OLM_MERGE, "pg_resetwal"},
+ {OLT_REWIND, OLM_MERGE, "pg_rewind"},
+ {OLT_UPGRADE, OLM_INSERT, "pg_upgrade"},
+ {OLT_PROMOTED, OLM_INSERT, "promoted"}
+};
+
+
+/*
+ * calculate_operation_log_crc()
+ *
+ * Calculate CRC of operation log.
+ */
+static pg_crc32c
+calculate_operation_log_crc(OperationLogBuffer * log_buffer)
+{
+ pg_crc32c crc;
+
+ INIT_CRC32C(crc);
+ COMP_CRC32C(crc,
+ (char *) log_buffer + sizeof(pg_crc32c),
+ PG_OPERATION_LOG_FULL_SIZE - sizeof(pg_crc32c));
+ FIN_CRC32C(crc);
+
+ return crc;
+}
+
+/*
+ * get_empty_operation_log()
+ *
+ * Function returns empty operation log buffer.
+ */
+OperationLogBuffer *
+get_empty_operation_log_buffer(void)
+{
+ OperationLogBuffer *log_buffer;
+
+ /* Initialize operation log file with zeros. */
+ log_buffer = palloc0(PG_OPERATION_LOG_FULL_SIZE);
+
+ /* Calculate CRC. */
+ log_buffer->header.ol_crc = calculate_operation_log_crc(log_buffer);
+
+ return log_buffer;
+}
+
+/*
+ * create_operation_log_file()
+ *
+ * Create file for operation log and initialize it with zeros.
+ * Function returns descriptor of created file or -1 in error case.
+ * Function cannot generate report with ERROR and FATAL for correct lock
+ * releasing on top level.
+ */
+static int
+create_operation_log_file(char *ControlLogFilePath)
+{
+ int fd;
+ OperationLogBuffer *log_buffer;
+
+#ifndef FRONTEND
+ fd = OpenTransientFile(ControlLogFilePath, O_RDWR | O_CREAT | PG_BINARY);
+
+ if (fd < 0)
+ {
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not create file \"%s\": %m",
+ PG_OPERATION_LOG_FILE)));
+ return -1;
+ }
+#else
+ fd = open(ControlLogFilePath, O_RDWR | O_CREAT | PG_BINARY, pg_file_create_mode);
+
+ if (fd < 0)
+ pg_fatal("could not create file \"%s\": %m",
+ ControlLogFilePath);
+#endif
+
+ /* Initialize operation log file with zeros. */
+ log_buffer = get_empty_operation_log_buffer();
+
+ errno = 0;
+ if (write(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE) != PG_OPERATION_LOG_FULL_SIZE)
+ {
+ /* If write didn't set errno, assume problem is no disk space. */
+ if (errno == 0)
+ errno = ENOSPC;
+
+#ifndef FRONTEND
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePath);
+#endif
+ pfree(log_buffer);
+ return -1;
+ }
+
+ pfree(log_buffer);
+
+#ifndef FRONTEND
+ if (pg_fsync(fd) != 0)
+ {
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not fsync file \"%s\": %m",
+ ControlLogFilePath)));
+ return -1;
+ }
+#else
+ if (fsync(fd) != 0)
+ pg_fatal("could not fsync file \"%s\": %m", ControlLogFilePath);
+#endif
+
+ if (lseek(fd, 0, SEEK_SET) != 0)
+ {
+#ifndef FRONTEND
+ ereport(LOG,
+ (errcode_for_file_access(),
+ errmsg("could not seek to position 0 of file \"%s\": %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not seek to position 0 of file \"%s\": %m",
+ ControlLogFilePath);
+#endif
+ return -1;
+ }
+
+ return fd;
+}
+
+#define LWLockReleaseSaveErrno(lock) \
+ save_errno = errno; \
+ LWLockRelease(lock); \
+ errno = save_errno; \
+
+/*
+ * get_operation_log()
+ *
+ * Get the operation log ring buffer. The result is returned as a palloc'd copy
+ * of operation log buffer.
+ *
+ * crc_ok_p can be used by the caller to see whether the CRC of the operation
+ * log is correct.
+ */
+OperationLogBuffer *
+get_operation_log(const char *DataDir, bool *crc_ok_p)
+{
+ OperationLogBuffer *log_buffer = NULL;
+ int fd;
+ char ControlLogFilePath[MAXPGPATH];
+ pg_crc32c crc;
+ int r;
+#ifndef FRONTEND
+ int save_errno;
+#endif
+
+ Assert(crc_ok_p);
+
+ snprintf(ControlLogFilePath, MAXPGPATH, "%s/%s", DataDir, PG_OPERATION_LOG_FILE);
+
+#ifndef FRONTEND
+ LWLockAcquire(ControlLogFileLock, LW_EXCLUSIVE);
+ fd = OpenTransientFile(ControlLogFilePath, O_RDONLY | PG_BINARY);
+#else
+ fd = open(ControlLogFilePath, O_RDONLY | PG_BINARY, 0);
+#endif
+ if (fd < 0)
+ {
+ /* File doesn't exist - try to create it. */
+ if (errno == ENOENT)
+ fd = create_operation_log_file(ControlLogFilePath);
+
+ if (fd < 0)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\" for reading: %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not open file \"%s\" for reading: %m",
+ ControlLogFilePath);
+#endif
+ }
+ }
+
+ log_buffer = palloc(PG_OPERATION_LOG_FULL_SIZE);
+
+ r = read(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE);
+ if (r != PG_OPERATION_LOG_FULL_SIZE)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("could not read operation log from the file \"%s\": read %d of %d",
+ ControlLogFilePath, r, PG_OPERATION_LOG_FULL_SIZE)));
+#else
+ pg_fatal("could not read operation log from the file \"%s\": read %d of %d",
+ ControlLogFilePath, r, PG_OPERATION_LOG_FULL_SIZE);
+#endif
+ }
+
+#ifndef FRONTEND
+ if (CloseTransientFile(fd) != 0)
+ {
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not close file \"%s\": %m",
+ ControlLogFilePath)));
+ }
+ LWLockRelease(ControlLogFileLock);
+#else
+ if (close(fd) != 0)
+ pg_fatal("could not close file \"%s\": %m", ControlLogFilePath);
+#endif
+
+ /* Check the CRC. */
+ crc = calculate_operation_log_crc(log_buffer);
+
+ *crc_ok_p = EQ_CRC32C(crc, log_buffer->header.ol_crc);
+
+ return log_buffer;
+}
+
+/*
+ * update_operation_log()
+ *
+ * Update the operation log ring buffer.
+ * Note. To protect against failures a operation log file is written in two
+ * stages: first a temporary file is created, then the temporary file is
+ * renamed to the operation log file.
+ */
+void
+update_operation_log(const char *DataDir, OperationLogBuffer * log_buffer)
+{
+ int fd;
+ char ControlLogFilePath[MAXPGPATH];
+ char ControlLogFilePathTmp[MAXPGPATH];
+#ifndef FRONTEND
+ int save_errno;
+#endif
+
+ snprintf(ControlLogFilePath, sizeof(ControlLogFilePath), "%s/%s", DataDir,
+ PG_OPERATION_LOG_FILE);
+ snprintf(ControlLogFilePathTmp, sizeof(ControlLogFilePathTmp), "%s.tmp",
+ ControlLogFilePath);
+
+ /* Recalculate CRC of operation log. */
+ log_buffer->header.ol_crc = calculate_operation_log_crc(log_buffer);
+
+#ifndef FRONTEND
+ LWLockAcquire(ControlLogFileLock, LW_EXCLUSIVE);
+#endif
+
+ /* Create a temporary file. */
+ fd = create_operation_log_file(ControlLogFilePathTmp);
+ if (fd < 0)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m",
+ ControlLogFilePathTmp)));
+#else
+ pg_fatal("could not open file \"%s\": %m", ControlLogFilePathTmp);
+#endif
+ }
+
+ /* Place operation log buffer into temporary file. */
+ errno = 0;
+ if (write(fd, log_buffer, PG_OPERATION_LOG_FULL_SIZE) != PG_OPERATION_LOG_FULL_SIZE)
+ {
+ /* if write didn't set errno, assume problem is no disk space */
+ if (errno == 0)
+ errno = ENOSPC;
+
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePathTmp)));
+#else
+ pg_fatal("could not write operation log in the file \"%s\": %m",
+ ControlLogFilePathTmp);
+#endif
+ }
+
+ /* Close the temporary file. */
+#ifndef FRONTEND
+ if (CloseTransientFile(fd) != 0)
+ {
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not close file \"%s\": %m",
+ ControlLogFilePathTmp)));
+ }
+#else
+ if (close(fd) != 0)
+ pg_fatal("could not close file \"%s\": %m", ControlLogFilePathTmp);
+#endif
+
+ /* Unlink old file with operation log. */
+ if (unlink(ControlLogFilePath) != 0)
+ {
+ /* File can be not exist: ignore this specific error. */
+ if (errno != ENOENT)
+ {
+#ifndef FRONTEND
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not unlink file \"%s\": %m",
+ ControlLogFilePath)));
+#else
+ pg_fatal("could not unlink file \"%s\": %m", ControlLogFilePath);
+#endif
+ }
+ }
+
+ /* Rename temporary file to required name. */
+#ifndef FRONTEND
+ if (durable_rename(ControlLogFilePathTmp, ControlLogFilePath, LOG) != 0)
+ {
+ LWLockReleaseSaveErrno(ControlLogFileLock);
+
+ ereport(PANIC,
+ (errcode_for_file_access(),
+ errmsg("could not rename file \"%s\" to \"%s\": %m",
+ ControlLogFilePathTmp, ControlLogFilePath)));
+ }
+ LWLockRelease(ControlLogFileLock);
+#else
+ if (durable_rename(ControlLogFilePathTmp, ControlLogFilePath) != 0)
+ pg_fatal("could not rename file \"%s\" to \"%s\": %m",
+ ControlLogFilePathTmp, ControlLogFilePath);
+#endif
+}
+
+/*
+ * is_enum_value_correct()
+ *
+ * Function returns true in case value is correct value of enum.
+ *
+ * val - test value;
+ * minval - first enum value (correct value);
+ * maxval - last enum value (incorrect value).
+ */
+static bool
+is_enum_value_correct(int16 val, int16 minval, int16 maxval)
+{
+ Assert(val >= minval || val < maxval);
+
+ if (val < minval || val >= maxval)
+ return false;
+ return true;
+}
+
+/*
+ * get_operation_log_type_desc()
+ *
+ * Function returns pointer to OperationLogTypeDesc struct for given type of
+ * operation ol_type.
+ */
+static OperationLogTypeDesc *
+get_operation_log_type_desc(ol_type_enum ol_type)
+{
+ return &OperationLogTypesDescs[ol_type - 1];
+}
+
+/*
+ * fill_operation_log_element()
+ *
+ * Fill new operation log element. Value of ol_lsn is last checkpoint record
+ * pointer.
+ */
+static void
+fill_operation_log_element(ControlFileData *ControlFile,
+ OperationLogTypeDesc * desc,
+ PgNumEdition edition, uint32 version_num,
+ OperationLogData * data)
+{
+ data->ol_type = desc->ol_type;
+ data->ol_edition = edition;
+ data->ol_count = 1;
+ data->ol_version = version_num;
+ data->ol_timestamp = (pg_time_t) time(NULL);
+ data->ol_lsn = ControlFile->checkPoint;
+}
+
+/*
+ * find_operation_log_element_for_merge()
+ *
+ * Find element into operation log ring buffer by ol_type and version.
+ * Returns NULL in case element is not found.
+ */
+static OperationLogData *
+find_operation_log_element_for_merge(ol_type_enum ol_type,
+ OperationLogBuffer * log_buffer,
+ PgNumEdition edition, uint32 version_num)
+{
+ uint32 first = log_buffer->header.ol_first;
+ uint32 count = get_operation_log_count(log_buffer);
+ OperationLogData *data;
+ uint32 i;
+
+ Assert(first < PG_OPERATION_LOG_COUNT && count <= PG_OPERATION_LOG_COUNT);
+
+ for (i = 0; i < count; i++)
+ {
+ data = &log_buffer->data[(first + i) % PG_OPERATION_LOG_COUNT];
+ if (data->ol_type == ol_type &&
+ data->ol_edition == edition &&
+ data->ol_version == version_num)
+ return data;
+ }
+
+ return NULL;
+}
+
+/*
+ * put_operation_log_element(), put_operation_log_element_version()
+ *
+ * Put element into operation log ring buffer.
+ *
+ * DataDir is the path to the top level of the PGDATA directory tree;
+ * ol_type is type of operation;
+ * edition is edition of current PostgreSQL version;
+ * version_num is number of version (for example 13000802 for v13.8.2).
+ *
+ * Note that it is up to the caller to properly lock ControlFileLogLock when
+ * calling this routine in the backend.
+ */
+void
+put_operation_log_element_version(const char *DataDir, ol_type_enum ol_type,
+ PgNumEdition edition, uint32 version_num)
+{
+ OperationLogBuffer *log_buffer;
+ ControlFileData *ControlFile;
+ bool crc_ok;
+ OperationLogTypeDesc *desc;
+
+ if (!is_enum_value_correct(ol_type, OLT_BOOTSTRAP, OLT_NumberOfTypes))
+ {
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("invalid type of operation (%u) for operation log", ol_type)));
+#else
+ pg_fatal("invalid type of operation (%u) for operation log", ol_type);
+#endif
+ }
+
+ desc = get_operation_log_type_desc(ol_type);
+
+ if (!is_enum_value_correct(desc->ol_mode, OLM_MERGE, OLM_NumberOfModes))
+ {
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("invalid mode of operation (%u) for operation log", ol_type)));
+#else
+ pg_fatal("invalid mode of operation (%u) for operation log", ol_type);
+#endif
+ }
+
+ /* get a copy of the control file */
+ ControlFile = get_controlfile(DataDir, &crc_ok);
+ if (!crc_ok)
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("pg_control CRC value is incorrect")));
+#else
+ pg_fatal("pg_control CRC value is incorrect");
+#endif
+
+ /* get a copy of the operation log */
+ log_buffer = get_operation_log(DataDir, &crc_ok);
+ if (!crc_ok)
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("pg_control_log CRC value is incorrect")));
+#else
+ pg_fatal("pg_control_log CRC value is incorrect");
+#endif
+
+ switch (desc->ol_mode)
+ {
+ case OLM_MERGE:
+ {
+ OperationLogData *data;
+
+ data = find_operation_log_element_for_merge(ol_type, log_buffer,
+ edition, version_num);
+ if (data)
+ {
+ /*
+ * We just found the element with the same type and the
+ * same version. Update it.
+ */
+ if (data->ol_count < PG_UINT16_MAX) /* prevent overflow */
+ data->ol_count++;
+ data->ol_timestamp = (pg_time_t) time(NULL);
+ data->ol_lsn = ControlFile->checkPoint;
+ break;
+ }
+ }
+ /* FALLTHROUGH */
+
+ case OLM_INSERT:
+ {
+ uint16 first = log_buffer->header.ol_first;
+ uint16 count = log_buffer->header.ol_count;
+ uint16 current;
+
+ Assert(first < PG_OPERATION_LOG_COUNT && count <= PG_OPERATION_LOG_COUNT);
+
+ if (count == PG_OPERATION_LOG_COUNT)
+ {
+ current = first;
+ /* Owerflow, shift the first element */
+ log_buffer->header.ol_first = (first + 1) % PG_OPERATION_LOG_COUNT;
+ }
+ else
+ {
+ current = first + count;
+ /* Increase number of elements: */
+ log_buffer->header.ol_count++;
+ }
+
+ /* Fill operation log element. */
+ fill_operation_log_element(ControlFile, desc, edition, version_num,
+ &log_buffer->data[current]);
+ break;
+ }
+
+ default:
+#ifndef FRONTEND
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("unexpected operation log mode %d",
+ desc->ol_mode)));
+#else
+ pg_fatal("unexpected operation log mode %d", desc->ol_mode);
+#endif
+ }
+
+ update_operation_log(DataDir, log_buffer);
+
+ pfree(log_buffer);
+
+ pfree(ControlFile);
+}
+
+/*
+ * Helper constant for determine current edition.
+ * Here can be custom editions.
+ */
+static const uint8 current_edition = ED_PG_ORIGINAL;
+
+/*
+ * Helper constant for determine current version.
+ * Multiplier 100 used as reserve of last two digits for patch number.
+ */
+static const uint32 current_version_num = PG_VERSION_NUM * 100;
+
+void
+put_operation_log_element(const char *DataDir, ol_type_enum ol_type)
+{
+ put_operation_log_element_version(DataDir, ol_type, current_edition, current_version_num);
+}
+
+/*
+ * get_operation_log_element()
+ *
+ * Returns operation log buffer element with number num.
+ */
+OperationLogData *
+get_operation_log_element(OperationLogBuffer * log_buffer, uint16 num)
+{
+ uint32 first = log_buffer->header.ol_first;
+#ifdef USE_ASSERT_CHECKING
+ uint32 count = get_operation_log_count(log_buffer);
+
+ Assert(num < count);
+#endif
+
+ return &log_buffer->data[(first + num) % PG_OPERATION_LOG_COUNT];
+}
+
+/*
+ * get_operation_log_count()
+ *
+ * Returns number of elements in given operation log buffer.
+ */
+uint16
+get_operation_log_count(OperationLogBuffer * log_buffer)
+{
+ return log_buffer->header.ol_count;
+}
+
+/*
+ * get_operation_log_type_name()
+ *
+ * Returns name of given type.
+ */
+const char *
+get_operation_log_type_name(ol_type_enum ol_type)
+{
+ if (is_enum_value_correct(ol_type, OLT_BOOTSTRAP, OLT_NumberOfTypes))
+ return OperationLogTypesDescs[ol_type - 1].ol_name;
+ else
+ return psprintf("unknown name %u", ol_type);
+}
diff --git a/src/common/meson.build b/src/common/meson.build
index 41bd58ebdf..2516913edd 100644
--- a/src/common/meson.build
+++ b/src/common/meson.build
@@ -6,6 +6,7 @@ common_sources = files(
'checksum_helper.c',
'compression.c',
'controldata_utils.c',
+ 'controllog_utils.c',
'encnames.c',
'exec.c',
'file_perm.c',
diff --git a/src/include/catalog/pg_controllog.h b/src/include/catalog/pg_controllog.h
new file mode 100644
index 0000000000..fddac25fea
--- /dev/null
+++ b/src/include/catalog/pg_controllog.h
@@ -0,0 +1,142 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_controllog.h
+ * The system operation log file "pg_control_log" is not a heap
+ * relation.
+ * However, we define it here so that the format is documented.
+ *
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/catalog/pg_controllog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PG_CONTROLLOG_H
+#define PG_CONTROLLOG_H
+
+#include "access/transam.h"
+#include "access/xlogdefs.h"
+#include "pgtime.h" /* for pg_time_t */
+#include "port/pg_crc32c.h"
+
+#define PG_OPERATION_LOG_FILE "global/pg_control_log"
+
+/*
+ * Type of operation for operation log.
+ */
+typedef enum
+{
+ OLT_BOOTSTRAP = 1, /* bootstrap */
+ OLT_STARTUP, /* server startup */
+ OLT_RESETWAL, /* pg_resetwal */
+ OLT_REWIND, /* pg_rewind */
+ OLT_UPGRADE, /* pg_upgrade */
+ OLT_PROMOTED, /* promoted */
+ OLT_NumberOfTypes /* should be last */
+} ol_type_enum;
+
+/*
+ * Mode of operation processing.
+ */
+typedef enum
+{
+ OLM_MERGE = 1, /* insert element only if not exists element
+ * with the same ol_type and ol_version;
+ * otherwise update existing element */
+ OLM_INSERT, /* insert element into ring buffer 'as is' */
+ OLM_NumberOfModes /* should be last */
+} ol_mode_enum;
+
+/*
+ * Helper struct for describing supported operations.
+ */
+typedef struct OperationLogTypeDesc
+{
+ ol_type_enum ol_type; /* element type */
+ ol_mode_enum ol_mode; /* element mode */
+ const char *ol_name; /* display name of element */
+} OperationLogTypeDesc;
+
+/*
+ * Element of operation log ring buffer (24 bytes).
+ */
+typedef struct OperationLogData
+{
+ uint8 ol_type; /* operation type */
+ uint8 ol_edition; /* postgres edition */
+ uint16 ol_count; /* number of operations */
+ uint32 ol_version; /* postgres version */
+ pg_time_t ol_timestamp; /* = int64, operation date/time */
+ XLogRecPtr ol_lsn; /* = uint64, last check point record ptr */
+} OperationLogData;
+
+/*
+ * Header of operation log ring buffer (8 bytes).
+ */
+typedef struct OperationLogHeader
+{
+ pg_crc32c ol_crc; /* CRC of operation log ... MUST BE FIRST! */
+ uint16 ol_first; /* position of first ring buffer element */
+ uint16 ol_count; /* number of elements in ring buffer */
+} OperationLogHeader;
+
+/*
+ * Whole size of the operation log ring buffer (with header).
+ */
+#define PG_OPERATION_LOG_FULL_SIZE 8192
+
+/*
+ * Size of elements of the operation log ring buffer.
+ * Value must be a multiple of sizeof(OperationLogData).
+ */
+#define PG_OPERATION_LOG_SIZE (PG_OPERATION_LOG_FULL_SIZE - sizeof(OperationLogHeader))
+
+/*
+ * Number of elements in the operation log.
+ */
+#define PG_OPERATION_LOG_COUNT (PG_OPERATION_LOG_SIZE / sizeof(OperationLogData))
+
+/*
+ * Operation log ring buffer.
+ */
+typedef struct OperationLogBuffer
+{
+ OperationLogHeader header;
+ OperationLogData data[PG_OPERATION_LOG_COUNT];
+
+} OperationLogBuffer;
+
+StaticAssertDecl(sizeof(OperationLogBuffer) == PG_OPERATION_LOG_FULL_SIZE,
+ "structure OperationLogBuffer must have size PG_OPERATION_LOG_FULL_SIZE");
+
+/* Enum for postgres edition. */
+typedef enum
+{
+ ED_PG_ORIGINAL = 0
+ /* Here can be custom editions */
+} PgNumEdition;
+
+#define ED_PG_ORIGINAL_STR "vanilla"
+#define ED_UNKNOWN_STR "unknown"
+
+/*
+ * get_str_edition()
+ *
+ * Returns edition string by edition number.
+ */
+static inline const char *
+get_str_edition(PgNumEdition edition)
+{
+ switch (edition)
+ {
+ case ED_PG_ORIGINAL:
+ return ED_PG_ORIGINAL_STR;
+
+ /* Here can be custom editions */
+ }
+ return ED_UNKNOWN_STR;
+}
+
+#endif /* PG_CONTROLLOG_H */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 3810de7b22..37397f2c86 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11891,4 +11891,13 @@
prorettype => 'bytea', proargtypes => 'pg_brin_minmax_multi_summary',
prosrc => 'brin_minmax_multi_summary_send' },
+# operation log function
+{ oid => '8110', descr => 'show operation log',
+ proname => 'pg_operation_log', prorows => '170', proretset => 't',
+ provolatile => 'v', prorettype => 'record', proargtypes => '',
+ proallargtypes => '{text,text,text,pg_lsn,timestamptz,int4}',
+ proargmodes => '{o,o,o,o,o,o}',
+ proargnames => '{event,edition,version,lsn,last,count}',
+ prosrc => 'pg_operation_log' },
+
]
diff --git a/src/include/common/controllog_utils.h b/src/include/common/controllog_utils.h
new file mode 100644
index 0000000000..dc5c01e87f
--- /dev/null
+++ b/src/include/common/controllog_utils.h
@@ -0,0 +1,27 @@
+/*
+ * controllog_utils.h
+ * Common code for pg_control_log output
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/controllog_utils.h
+ */
+#ifndef COMMON_CONTROLLOG_UTILS_H
+#define COMMON_CONTROLLOG_UTILS_H
+
+#include "catalog/pg_controllog.h"
+
+extern OperationLogBuffer * get_operation_log(const char *DataDir, bool *crc_ok_p);
+extern OperationLogBuffer * get_empty_operation_log_buffer(void);
+extern void update_operation_log(const char *DataDir, OperationLogBuffer * log_buffer);
+
+extern void put_operation_log_element(const char *DataDir, ol_type_enum ol_type);
+extern void put_operation_log_element_version(const char *DataDir, ol_type_enum ol_type,
+ PgNumEdition edition, uint32 version_num);
+extern uint16 get_operation_log_count(OperationLogBuffer * log_buffer);
+extern OperationLogData * get_operation_log_element(OperationLogBuffer * log_buffer,
+ uint16 num);
+extern const char *get_operation_log_type_name(ol_type_enum ol_type);
+
+#endif /* COMMON_CONTROLLOG_UTILS_H */
diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build
index 21bde427b4..ec698284af 100644
--- a/src/test/modules/test_misc/meson.build
+++ b/src/test/modules/test_misc/meson.build
@@ -9,6 +9,7 @@ tests += {
't/001_constraint_validation.pl',
't/002_tablespace.pl',
't/003_check_guc.pl',
+ 't/004_operation_log.pl',
],
},
}
diff --git a/src/test/modules/test_misc/t/004_operation_log.pl b/src/test/modules/test_misc/t/004_operation_log.pl
new file mode 100644
index 0000000000..3a07f78ffa
--- /dev/null
+++ b/src/test/modules/test_misc/t/004_operation_log.pl
@@ -0,0 +1,136 @@
+
+# Copyright (c) 2022, PostgreSQL Global Development Group
+
+# Test for operation log.
+#
+# Some events like
+# "bootstrap", "startup", "pg_rewind", "pg_resetwal", "promoted", "pg_upgrade"
+# should be registered in operation log.
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+use File::Copy;
+
+# Create and start primary node
+my $node_primary = PostgreSQL::Test::Cluster->new('primary');
+$node_primary->init(allows_streaming => 1);
+$node_primary->append_conf('postgresql.conf', qq(wal_keep_size = 100MB));
+$node_primary->start;
+
+# Get server version
+my $server_version = $node_primary->safe_psql("postgres", "SELECT current_setting('server_version_num');") + 0;
+my $major_version = $server_version / 10000;
+my $minor_version = $server_version % 100;
+
+# Get primary node backup
+$node_primary->backup('primary_backup');
+
+# Initialize standby node from backup
+my $node_standby = PostgreSQL::Test::Cluster->new('standby');
+$node_standby->init_from_backup($node_primary, 'primary_backup', has_streaming => 1);
+$node_standby->start;
+
+# Promote the standby
+$node_standby->promote;
+
+# Stop standby node
+$node_standby->stop;
+
+my $node_standby_pgdata = $node_standby->data_dir;
+my $node_primary_connstr = $node_primary->connstr;
+
+# Keep a temporary postgresql.conf or it would be overwritten during the rewind.
+my $tmp_folder = PostgreSQL::Test::Utils::tempdir;
+copy("$node_standby_pgdata/postgresql.conf", "$tmp_folder/node_standby-postgresql.conf.tmp");
+
+# Get "pg_rewind" event
+command_ok(
+ [
+ 'pg_rewind',
+ "--source-server=$node_primary_connstr",
+ "--target-pgdata=$node_standby_pgdata",
+ "--debug"
+ ],
+ 'run pg_rewind');
+
+# Move back postgresql.conf with old settings
+move("$tmp_folder/node_standby-postgresql.conf.tmp", "$node_standby_pgdata/postgresql.conf");
+
+# Start and stop standby before resetwal and upgrade
+$node_standby->start;
+$node_standby->stop;
+
+# Get first "pg_resetwal" event
+system_or_bail('pg_resetwal', '-f', $node_standby->data_dir);
+
+# Get second "pg_resetwal" event
+system_or_bail('pg_resetwal', '-f', $node_standby->data_dir);
+
+# Initialize a new node for the upgrade
+my $node_new = PostgreSQL::Test::Cluster->new('new');
+$node_new->init;
+
+my $bindir_new = $node_new->config_data('--bindir');
+my $bindir_standby = $node_standby->config_data('--bindir');
+
+# We want to run pg_upgrade in the build directory so that any files generated
+# finish in it, like delete_old_cluster.{sh,bat}.
+chdir ${PostgreSQL::Test::Utils::tmp_check};
+
+# Run pg_upgrade
+command_ok(
+ [
+ 'pg_upgrade', '--no-sync', '-d', $node_standby->data_dir,
+ '-D', $node_new->data_dir, '-b', $bindir_standby,
+ '-B', $bindir_new, '-s', $node_new->host,
+ '-p', $node_standby->port, '-P', $node_new->port
+ ],
+ 'run pg_upgrade');
+#
+# Need to check operation log
+#
+sub check_event
+{
+ my $event_name = shift;
+ my $result = shift;
+ my $func_args = shift ? "sum(count), count(*)" : "count(*)";
+
+ my $psql_stdout = $node_new->safe_psql('postgres', qq(
+ SELECT
+ $func_args,
+ min(split_part(version,'.','1')),
+ min(split_part(version,'.','2'))
+ FROM
+ pg_operation_log()
+ WHERE
+ event='$event_name'));
+
+ is($psql_stdout, $result, 'check number of event ' . $event_name);
+ return;
+}
+#Start new node
+$node_new->start;
+
+# Check number of event "bootstrap"
+check_event('bootstrap', qq(1|1|$major_version|$minor_version), 1);
+
+# Check number of event "startup"
+check_event('startup', qq(1|$major_version|$minor_version), 0);
+
+# Check number of event "promoted"
+check_event('promoted', qq(1|1|$major_version|$minor_version), 1);
+
+# Check number of event "pg_upgrade"
+check_event('pg_upgrade', qq(1|1|$major_version|$minor_version), 1);
+
+# Check number of event "pg_resetwal"
+check_event('pg_resetwal', qq(2|1|$major_version|$minor_version), 1);
+
+# Check number of event "pg_rewind"
+check_event('pg_rewind', qq(1|1|$major_version|$minor_version), 1);
+
+done_testing();
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index f1c9ddf4a0..0cfd96d9ba 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -134,8 +134,8 @@ sub mkvcbuild
our @pgcommonallfiles = qw(
archive.c base64.c checksum_helper.c compression.c
- config_info.c controldata_utils.c d2s.c encnames.c exec.c
- f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
+ config_info.c controldata_utils.c controllog_utils.c d2s.c encnames.c
+ exec.c f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
keywords.c kwlookup.c link-canary.c md5_common.c percentrepl.c
pg_get_line.c pg_lzcompress.c pg_prng.c pgfnames.c psprintf.c relpath.c
rmtree.c saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c
--
2.31.0.windows.1
view thread (3+ messages)
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected], [email protected], [email protected]
Subject: Re: Operation log for major operations
In-Reply-To: <[email protected]>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox