From d0e79b6152342cef755c946e6690d2409cb78de6 Mon Sep 17 00:00:00 2001
From: David Christensen <david.christensen@crunchydata.com>
Date: Tue, 31 Oct 2023 15:24:17 -0400
Subject: [PATCH v3 4/5] bin tools and programs

---
 src/bin/Makefile                        |   1 +
 src/bin/initdb/initdb.c                 | 119 +++-
 src/bin/meson.build                     |   1 +
 src/bin/pg_alterckey/.gitignore         |   1 +
 src/bin/pg_alterckey/Makefile           |  38 ++
 src/bin/pg_alterckey/README             |  24 +
 src/bin/pg_alterckey/meson.build        |  16 +
 src/bin/pg_alterckey/pg_alterckey.c     | 792 ++++++++++++++++++++++++
 src/bin/pg_controldata/pg_controldata.c |   3 +
 src/bin/pg_ctl/pg_ctl.c                 |  59 +-
 src/bin/pg_resetwal/pg_resetwal.c       |   3 +
 src/bin/pg_rewind/filemap.c             |   8 +
 src/bin/pg_upgrade/check.c              |  34 +
 src/bin/pg_upgrade/controldata.c        |  41 +-
 src/bin/pg_upgrade/file.c               |   2 +
 src/bin/pg_upgrade/option.c             |   7 +-
 src/bin/pg_upgrade/pg_upgrade.h         |   4 +
 src/bin/pg_upgrade/server.c             |   2 +-
 18 files changed, 1137 insertions(+), 18 deletions(-)
 create mode 100644 src/bin/pg_alterckey/.gitignore
 create mode 100644 src/bin/pg_alterckey/Makefile
 create mode 100644 src/bin/pg_alterckey/README
 create mode 100644 src/bin/pg_alterckey/meson.build
 create mode 100644 src/bin/pg_alterckey/pg_alterckey.c

diff --git a/src/bin/Makefile b/src/bin/Makefile
index 373077bf52..9488f5fd9e 100644
--- a/src/bin/Makefile
+++ b/src/bin/Makefile
@@ -17,6 +17,7 @@ SUBDIRS = \
 	initdb \
 	pg_amcheck \
 	pg_archivecleanup \
+	pg_alterckey \
 	pg_basebackup \
 	pg_checksums \
 	pg_config \
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 0c6f5ceb0a..b676e5d2c9 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -71,6 +71,7 @@
 #include "catalog/pg_database_d.h"	/* pgrminclude ignore */
 #include "common/file_perm.h"
 #include "common/file_utils.h"
+#include "common/kmgr_utils.h"
 #include "common/logging.h"
 #include "common/pg_prng.h"
 #include "common/restricted_token.h"
@@ -161,11 +162,16 @@ static bool noclean = false;
 static bool noinstructions = false;
 static bool do_sync = true;
 static bool sync_only = false;
+static bool pass_terminal_fd = false;
+static char *term_fd_opt = NULL;
+static int file_encryption_method = DISABLED_ENCRYPTION_METHOD;
 static bool show_setting = false;
 static bool data_checksums = false;
 static char *xlog_dir = NULL;
 static int	wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024);
 static DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
+static char *cluster_key_cmd = NULL;
+static char *old_key_datadir = NULL;
 
 
 /* internal vars */
@@ -228,6 +234,7 @@ static const char *const subdirs[] = {
 	"global",
 	"pg_wal/archive_status",
 	"pg_commit_ts",
+	"pg_cryptokeys",
 	"pg_dynshmem",
 	"pg_notify",
 	"pg_serial",
@@ -1148,11 +1155,12 @@ test_specific_config_settings(int test_conns, int test_buffs)
 
 	/* Set up the test postmaster invocation */
 	printfPQExpBuffer(&cmd,
-					  "\"%s\" --check %s %s "
+					  "\"%s\" --check %s %s %s"
 					  "-c max_connections=%d "
 					  "-c shared_buffers=%d "
 					  "-c dynamic_shared_memory_type=%s",
 					  backend_exec, boot_options, extra_options,
+					  term_fd_opt ? term_fd_opt : "",
 					  test_conns, test_buffs,
 					  dynamic_shared_memory_type);
 
@@ -1339,6 +1347,13 @@ setup_config(void)
 									  "md5", false);
 	}
 
+	if (cluster_key_cmd)
+	{
+		snprintf(repltok, sizeof(repltok), "cluster_key_command = '%s'",
+				 escape_quotes(cluster_key_cmd));
+		conflines = replace_token(conflines, "#cluster_key_command = ''", repltok);
+	}
+
 	/*
 	 * If group access has been enabled for the cluster then it makes sense to
 	 * ensure that the log files also allow group access.  Otherwise a backup
@@ -1540,6 +1555,12 @@ bootstrap_template1(void)
 	appendPQExpBuffer(&cmd, " -X %d", wal_segment_size_mb * (1024 * 1024));
 	if (data_checksums)
 		appendPQExpBuffer(&cmd, " -k");
+	if (cluster_key_cmd)
+		appendPQExpBuffer(&cmd, " -K %s", encryption_methods[file_encryption_method].name);
+	if (old_key_datadir)
+		appendPQExpBuffer(&cmd, " -u %s", old_key_datadir);
+	if (term_fd_opt)
+		appendPQExpBuffer(&cmd, " %s", term_fd_opt);
 	if (debug)
 		appendPQExpBuffer(&cmd, " -d 5");
 
@@ -2455,20 +2476,28 @@ usage(const char *progname)
 	printf(_("  -T, --text-search-config=CFG\n"
 			 "                            default text search configuration\n"));
 	printf(_("  -U, --username=NAME       database superuser name\n"));
-	printf(_("  -W, --pwprompt            prompt for a password for the new superuser\n"));
+	printf(_("  -W, --pwprompt            prompt for the new superuser password\n"));
 	printf(_("  -X, --waldir=WALDIR       location for the write-ahead log directory\n"));
 	printf(_("      --wal-segsize=SIZE    size of WAL segments, in megabytes\n"));
 	printf(_("\nLess commonly used options:\n"));
 	printf(_("  -c, --set NAME=VALUE      override default setting for server parameter\n"));
+	printf(_("      --cluster-key-command=COMMAND\n"
+			 "                            enable cluster file encryption and set command\n"
+			 "                            to obtain the cluster key\n"));
 	printf(_("  -d, --debug               generate lots of debugging output\n"));
 	printf(_("      --discard-caches      set debug_discard_caches=1\n"));
+	printf(_("  -K, --file-encryption-method=METHOD\n"
+			 "                            cluster file encryption method\n"));
 	printf(_("  -L DIRECTORY              where to find the input files\n"));
 	printf(_("  -n, --no-clean            do not clean up after errors\n"));
 	printf(_("  -N, --no-sync             do not wait for changes to be written safely to disk\n"));
+	printf(_("  -R, --authprompt          prompt for a passphrase or PIN\n"));
 	printf(_("      --no-instructions     do not print instructions for next steps\n"));
 	printf(_("  -s, --show                show internal settings\n"));
 	printf(_("      --sync-method=METHOD  set method for syncing files to disk\n"));
 	printf(_("  -S, --sync-only           only sync database files to disk, then exit\n"));
+	printf(_("  -u, --copy-encryption-keys=DATADIR\n"
+			 "                            copy the file encryption key from another cluster\n"));
 	printf(_("\nOther options:\n"));
 	printf(_("  -V, --version             output version information, then exit\n"));
 	printf(_("  -?, --help                show this help, then exit\n"));
@@ -3000,6 +3029,23 @@ initialize_data_directory(void)
 	/* Top level PG_VERSION is checked by bootstrapper, so make it first */
 	write_version_file(NULL);
 
+	if (pass_terminal_fd)
+	{
+#ifndef WIN32
+		int terminal_fd = open("/dev/tty", O_RDWR, 0);
+#else
+		int terminal_fd = open("CONOUT$", O_RDWR, 0);
+#endif
+
+		if (terminal_fd < 0)
+		{
+			pg_log_error(_("%s: could not open terminal: %s\n"),
+						 progname, strerror(errno));
+			exit(1);
+		}
+		term_fd_opt = psprintf("-R %d", terminal_fd);
+	}
+
 	/* Select suitable configuration settings */
 	set_null_conf();
 	test_config_settings();
@@ -3023,8 +3069,8 @@ initialize_data_directory(void)
 	fflush(stdout);
 
 	initPQExpBuffer(&cmd);
-	printfPQExpBuffer(&cmd, "\"%s\" %s %s template1 >%s",
-					  backend_exec, backend_options, extra_options, DEVNULL);
+	printfPQExpBuffer(&cmd, "\"%s\" %s %s %s template1 >%s",
+					  backend_exec, backend_options, extra_options, term_fd_opt ? term_fd_opt : "", DEVNULL);
 
 	PG_CMD_OPEN(cmd.data);
 
@@ -3103,12 +3149,16 @@ main(int argc, char *argv[])
 		{"waldir", required_argument, NULL, 'X'},
 		{"wal-segsize", required_argument, NULL, 12},
 		{"data-checksums", no_argument, NULL, 'k'},
+		{"authprompt", no_argument, NULL, 'R'},
+		{"file-encryption-method", required_argument, NULL, 'K'},
 		{"allow-group-access", no_argument, NULL, 'g'},
 		{"discard-caches", no_argument, NULL, 14},
 		{"locale-provider", required_argument, NULL, 15},
 		{"icu-locale", required_argument, NULL, 16},
 		{"icu-rules", required_argument, NULL, 17},
 		{"sync-method", required_argument, NULL, 18},
+		{"cluster-key-command", required_argument, NULL, 21},
+		{"copy-encryption-keys", required_argument, NULL, 'u'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -3150,7 +3200,7 @@ main(int argc, char *argv[])
 
 	/* process command-line options */
 
-	while ((c = getopt_long(argc, argv, "A:c:dD:E:gkL:nNsST:U:WX:",
+	while ((c = getopt_long(argc, argv, "A:c:dD:E:gkK:L:nNRsST:u:U:WX:",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
@@ -3215,6 +3265,28 @@ main(int argc, char *argv[])
 			case 'N':
 				do_sync = false;
 				break;
+			case 'R':
+				pass_terminal_fd = true;
+				break;
+			case 'K':
+				{
+					int i;
+
+					/* method 0/disabled cannot be specified */
+					for (i = DISABLED_ENCRYPTION_METHOD + 1;
+						 i < NUM_ENCRYPTION_METHODS; i++)
+						if (pg_strcasecmp(optarg, encryption_methods[i].name) == 0)
+						{
+							file_encryption_method = i;
+							break;
+						}
+					if (i == NUM_ENCRYPTION_METHODS)
+					{
+						fprintf(stderr, _("invalid cluster encryption method\n"));
+						exit(1);
+					}
+				}
+				break;
 			case 'S':
 				sync_only = true;
 				break;
@@ -3251,6 +3323,12 @@ main(int argc, char *argv[])
 			case 9:
 				pwfilename = pg_strdup(optarg);
 				break;
+			case 21:
+				cluster_key_cmd = pg_strdup(optarg);
+				break;
+			case 'u':
+				old_key_datadir = pg_strdup(optarg);
+				break;
 			case 's':
 				show_setting = true;
 				break;
@@ -3348,6 +3426,32 @@ main(int argc, char *argv[])
 	if (pwprompt && pwfilename)
 		pg_fatal("password prompt and password file cannot be specified together");
 
+#ifndef USE_OPENSSL
+	if (cluster_key_cmd)
+	{
+		pg_log_error("cluster file encryption is not supported because OpenSSL is not supported by this build");
+		exit(1);
+	}
+#endif
+
+	if (old_key_datadir != NULL && cluster_key_cmd == NULL)
+	{
+		pg_log_error("copying encryption keys requires the cluster key command to be specified");
+		exit(1);
+	}
+
+	if (file_encryption_method != DISABLED_ENCRYPTION_METHOD &&
+		cluster_key_cmd == NULL)
+	{
+		pg_log_error("a file encryption method requires the cluster key command to be specified");
+		exit(1);
+	}
+
+	/* set the default */
+	if (file_encryption_method == DISABLED_ENCRYPTION_METHOD &&
+		cluster_key_cmd != NULL)
+		file_encryption_method = DEFAULT_ENABLED_ENCRYPTION_METHOD;
+
 	check_authmethod_unspecified(&authmethodlocal);
 	check_authmethod_unspecified(&authmethodhost);
 
@@ -3392,6 +3496,11 @@ main(int argc, char *argv[])
 	else
 		printf(_("Data page checksums are disabled.\n"));
 
+	if (cluster_key_cmd)
+		printf(_("Cluster file encryption is enabled.\n"));
+	else
+		printf(_("Cluster file encryption is disabled.\n"));
+
 	if (pwprompt || pwfilename)
 		get_su_pwd();
 
diff --git a/src/bin/meson.build b/src/bin/meson.build
index 67cb50630c..434106f7c6 100644
--- a/src/bin/meson.build
+++ b/src/bin/meson.build
@@ -1,6 +1,7 @@
 # Copyright (c) 2022-2023, PostgreSQL Global Development Group
 
 subdir('initdb')
+subdir('pg_alterckey')
 subdir('pg_amcheck')
 subdir('pg_archivecleanup')
 subdir('pg_basebackup')
diff --git a/src/bin/pg_alterckey/.gitignore b/src/bin/pg_alterckey/.gitignore
new file mode 100644
index 0000000000..4c4f39f2cc
--- /dev/null
+++ b/src/bin/pg_alterckey/.gitignore
@@ -0,0 +1 @@
+/pg_alterckey
diff --git a/src/bin/pg_alterckey/Makefile b/src/bin/pg_alterckey/Makefile
new file mode 100644
index 0000000000..2133a4ba06
--- /dev/null
+++ b/src/bin/pg_alterckey/Makefile
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/bin/pg_alterckey
+#
+# Copyright (c) 1998-2021, PostgreSQL Global Development Group
+#
+# src/bin/pg_alterckey/Makefile
+#
+#-------------------------------------------------------------------------
+
+PGFILEDESC = "pg_alterckey - alter the cluster key"
+PGAPPICON=win32
+
+subdir = src/bin/pg_alterckey
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = \
+	$(WIN32RES) \
+	pg_alterckey.o
+
+all: pg_alterckey
+
+pg_alterckey: $(OBJS) | submake-libpgport
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+install: all installdirs
+	$(INSTALL_PROGRAM) pg_alterckey$(X) '$(DESTDIR)$(bindir)/pg_alterckey$(X)'
+
+installdirs:
+	$(MKDIR_P) '$(DESTDIR)$(bindir)'
+
+uninstall:
+	rm -f '$(DESTDIR)$(bindir)/pg_alterckey$(X)'
+
+clean distclean maintainer-clean:
+	rm -f pg_alterckey$(X) $(OBJS)
+	rm -rf tmp_check
diff --git a/src/bin/pg_alterckey/README b/src/bin/pg_alterckey/README
new file mode 100644
index 0000000000..6db3739271
--- /dev/null
+++ b/src/bin/pg_alterckey/README
@@ -0,0 +1,24 @@
+pg_alterckey
+============
+
+This directory contains the code to generate the pg_alterckey binary.
+
+Architecture
+------------
+
+pg_alterckey allows altering of the cluster encryption key (key
+encryption key or KEK) which is stored outside of the file system;  see
+src/backend/crypto/README for more details.  This must be done in a
+crash-safe manner since the keys are critical to reading an encrypted
+cluster.  The active data encryption keys (DEK) are encrypted/wrapped by
+the KEK and stored in PGDATA/pg_cryptokeys/live as separate files,
+currently files 0 and 1.
+
+This process can be interrupted at anytime;  the new execution of
+pg_alterckey will repair any previously interrupted execution of
+pg_alterckey.
+
+pg_alterckey should never be run concurrently.  A lock file prevents
+almost all concurrent execution.  pg_alterckey can be run if the
+database server is running or stopped, so it can't use database locking
+that is only available when the server is running.
diff --git a/src/bin/pg_alterckey/meson.build b/src/bin/pg_alterckey/meson.build
new file mode 100644
index 0000000000..72ec8274c3
--- /dev/null
+++ b/src/bin/pg_alterckey/meson.build
@@ -0,0 +1,16 @@
+pg_alterckey_sources = files(
+  'pg_alterckey.c',
+)
+
+if host_system == 'windows'
+  pg_alterckey_sources += rc_bin_gen.process(win32ver_rc, extra_args: [
+    '--NAME', 'pg_alterckey',
+    '--FILEDESC', 'pg_alterckey - alter the cluster key',])
+endif
+
+pg_alterckey = executable('pg_alterckey',
+  pg_alterckey_sources,
+  dependencies: [frontend_code, libpq],
+  kwargs: default_bin_args,
+)
+bin_targets += pg_alterckey
diff --git a/src/bin/pg_alterckey/pg_alterckey.c b/src/bin/pg_alterckey/pg_alterckey.c
new file mode 100644
index 0000000000..1aea340c7d
--- /dev/null
+++ b/src/bin/pg_alterckey/pg_alterckey.c
@@ -0,0 +1,792 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_alterckey.c
+ *    A utility to change the cluster key (key encryption key, KEK)
+ *    used for cluster file encryption.  The KEK wrap data encryption
+ *    keys (DEK).
+ *
+ * The theory of operation is fairly simple:
+ *    1. Create lock file
+ *    2. Retrieve current and new cluster key using the supplied
+ *       commands.
+ *    3. Revert any failed alter operation.
+ *    4. Create a "new" directory
+ *    5. Unwrap each DEK in "live" using the old KEK
+ *    6. Wrap each DEK using the new KEK and write it to "new"
+ *    7. Rename "live" to "old"
+ *    8. Rename "new" to "live"
+ *    9. Remove "old"
+ *   10. Remove lock file
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_alterckey/pg_alterckey.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+
+#define FRONTEND 1
+
+#include "postgres_fe.h"
+
+#include <signal.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "common/controldata_utils.h"
+#include "common/file_perm.h"
+#include "common/file_utils.h"
+#include "common/restricted_token.h"
+#include "common/logging.h"
+#include "crypto/kmgr.h"
+#include "getopt_long.h"
+#include "pg_getopt.h"
+
+typedef enum
+{
+	SUCCESS_EXIT = 0,
+	ERROR_EXIT,
+	RMDIR_EXIT,
+	REPAIR_EXIT
+} exit_action;
+
+#define MAX_WRAPPED_KEY_LENGTH 64
+
+static int	lock_fd = -1;
+static bool pass_terminal_fd = false;
+int			terminal_fd = -1;
+static bool repair_mode = false;
+static char *old_cluster_key_cmd = NULL,
+		   *new_cluster_key_cmd = NULL;
+static char old_cluster_key[KMGR_CLUSTER_KEY_MAX_LEN],
+	new_cluster_key[KMGR_CLUSTER_KEY_MAX_LEN];
+static CryptoKey data_key;
+unsigned char in_key[MAX_WRAPPED_KEY_LENGTH],
+			out_key[MAX_WRAPPED_KEY_LENGTH];
+int			in_klen,
+			out_klen;
+static char top_path[MAXPGPATH],
+			pid_path[MAXPGPATH],
+			live_path[MAXPGPATH],
+			new_path[MAXPGPATH],
+			old_path[MAXPGPATH];
+
+static char *DataDir = NULL;
+static const char *progname;
+static ControlFileData *ControlFile;
+
+static void create_lockfile(void);
+static void recover_failure(void);
+static void retrieve_cluster_keys(void);
+static void bzero_keys_and_exit(exit_action action);
+static void reencrypt_data_keys(void);
+static void install_new_keys(void);
+
+static uint64 hex_decode(const char *src, size_t len, char *dst);
+
+
+static void
+usage(const char *progname)
+{
+	printf(_("%s changes the cluster key of a PostgreSQL database cluster.\n\n"), progname);
+	printf(_("Usage:\n"));
+	printf(_("  %s [OPTION] old_cluster_key_command new_cluster_key_command [DATADIR]\n"), progname);
+	printf(_("  %s [repair_option] [DATADIR]\n"), progname);
+	printf(_("\nOptions:\n"));
+	printf(_("  -R, --authprompt       prompt for a passphrase or PIN\n"));
+	printf(_(" [-D, --pgdata=]DATADIR  data directory\n"));
+	printf(_("  -V, --version          output version information, then exit\n"));
+	printf(_("  -?, --help             show this help, then exit\n"));
+	printf(_("\nRepair options:\n"));
+	printf(_("  -r, --repair           repair previous failure\n"));
+	printf(_("\nIf no data directory (DATADIR) is specified, "
+			 "the environment variable PGDATA\nis used.\n\n"));
+	printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
+	printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
+}
+
+
+int
+main(int argc, char *argv[])
+{
+	static struct option long_options[] = {
+		{"authprompt", required_argument, NULL, 'R'},
+		{"repair", required_argument, NULL, 'r'},
+		{"pgdata", required_argument, NULL, 'D'},
+		{NULL, 0, NULL, 0}
+	};
+
+	int			c;
+
+	pg_logging_init(argv[0]);
+	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_alterckey"));
+	progname = get_progname(argv[0]);
+
+	if (argc > 1)
+	{
+		if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
+		{
+			usage(progname);
+			exit(0);
+		}
+		if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
+		{
+			puts("pg_alterckey (PostgreSQL) " PG_VERSION);
+			exit(0);
+		}
+	}
+
+	/* check for -r/-R */
+	while ((c = getopt_long(argc, argv, "D:rR", long_options, NULL)) != -1)
+	{
+		switch (c)
+		{
+			case 'D':
+				DataDir = optarg;
+				break;
+			case 'r':
+				repair_mode = true;
+				break;
+			case 'R':
+				pass_terminal_fd = true;
+				break;
+			default:
+				fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
+				exit(1);
+		}
+	}
+
+	if (!repair_mode)
+	{
+		/* get cluster key commands */
+		if (optind < argc)
+			old_cluster_key_cmd = argv[optind++];
+		else
+		{
+			pg_log_error("missing old_cluster_key_command");
+			fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+					progname);
+			exit(1);
+		}
+
+		if (optind < argc)
+			new_cluster_key_cmd = argv[optind++];
+		else
+		{
+			pg_log_error("missing new_cluster_key_command");
+			fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+					progname);
+			exit(1);
+		}
+	}
+
+	if (DataDir == NULL)
+	{
+		if (optind < argc)
+			DataDir = argv[optind++];
+		else
+			DataDir = getenv("PGDATA");
+
+		/* If no DataDir was specified, and none could be found, error out */
+		if (DataDir == NULL)
+		{
+			pg_log_error("no data directory specified");
+			fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
+			exit(1);
+		}
+	}
+
+	/* Complain if any arguments remain */
+	if (optind < argc)
+	{
+		pg_log_error("too many command-line arguments (first is \"%s\")",
+					 argv[optind]);
+		fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+				progname);
+		exit(1);
+	}
+
+	/*
+	 * Disallow running as root because we create directories in PGDATA
+	 */
+#ifndef WIN32
+	if (geteuid() == 0)
+	{
+		pg_log_error("%s: cannot be run as root\n"
+					 "Please log in (using, e.g., \"su\") as the "
+					 "(unprivileged) user that will\n"
+					 "own the server process.\n",
+					 progname);
+		exit(1);
+	}
+#endif
+
+	ControlFile = get_controlfile(DataDir, NULL);
+	
+	get_restricted_token();
+
+	/* Set mask based on PGDATA permissions */
+	if (!GetDataDirectoryCreatePerm(DataDir))
+	{
+		pg_log_error("could not read permissions of directory \"%s\": %m",
+					 DataDir);
+		exit(1);
+	}
+
+	umask(pg_mode_mask);
+
+	snprintf(top_path, sizeof(top_path), "%s/%s", DataDir, KMGR_DIR);
+	snprintf(pid_path, sizeof(pid_path), "%s/%s", DataDir, KMGR_DIR_PID);
+	snprintf(live_path, sizeof(live_path), "%s/%s", DataDir, LIVE_KMGR_DIR);
+	snprintf(new_path, sizeof(new_path), "%s/%s", DataDir, NEW_KMGR_DIR);
+	snprintf(old_path, sizeof(old_path), "%s/%s", DataDir, OLD_KMGR_DIR);
+
+	/* Complain if any arguments remain */
+	if (optind < argc)
+	{
+		pg_log_error("too many command-line arguments (first is \"%s\")",
+					 argv[optind]);
+		fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
+				progname);
+		exit(1);
+	}
+
+	if (DataDir == NULL)
+	{
+		pg_log_error("no data directory specified");
+		fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
+		exit(1);
+	}
+
+	create_lockfile();
+
+	recover_failure();
+
+	if (!repair_mode)
+	{
+		retrieve_cluster_keys();
+		reencrypt_data_keys();
+		install_new_keys();
+	}
+
+#ifndef WIN32
+	/* remove file system reference to file */
+	if (unlink(pid_path) < 0)
+	{
+		pg_log_error("could not delete lock file \"%s\": %m", KMGR_DIR_PID);
+		exit(1);
+	}
+#endif
+
+	close(lock_fd);
+
+	bzero_keys_and_exit(SUCCESS_EXIT);
+}
+
+/* Create a lock file;  this prevents almost all cases of concurrent access */
+void
+create_lockfile(void)
+{
+	struct stat buffer;
+	char		lock_pid_str[20];
+
+	if (stat(top_path, &buffer) != 0 || !S_ISDIR(buffer.st_mode))
+	{
+		pg_log_error("cluster file encryption directory \"%s\" is missing;  is it enabled?", KMGR_DIR_PID);
+		fprintf(stderr, _("Exiting with no changes made.\n"));
+		exit(1);
+	}
+
+	/* Does a lockfile exist? */
+	if ((lock_fd = open(pid_path, O_RDONLY, 0)) != -1)
+	{
+		int			lock_pid;
+		int			len;
+
+		/* read the PID */
+		if ((len = read(lock_fd, lock_pid_str, sizeof(lock_pid_str) - 1)) == 0)
+		{
+			pg_log_error("cannot read pid from lock file \"%s\": %m", KMGR_DIR_PID);
+			fprintf(stderr, _("Exiting with no changes made.\n"));
+			exit(1);
+		}
+		lock_pid_str[len] = '\0';
+
+		if ((lock_pid = atoi(lock_pid_str)) == 0)
+		{
+			pg_log_error("invalid pid in lock file \"%s\": %m", KMGR_DIR_PID);
+			fprintf(stderr, _("Exiting with no changes made.\n"));
+			exit(1);
+		}
+
+		/* Is the PID running? */
+		if (kill(lock_pid, 0) == 0)
+		{
+			pg_log_error("active process %d currently holds a lock on this operation, recorded in \"%s\"",
+						 lock_pid, KMGR_DIR_PID);
+			fprintf(stderr, _("Exiting with no changes made.\n"));
+			exit(1);
+		}
+
+		close(lock_fd);
+
+		if (repair_mode)
+			printf("old lock file removed\n");
+
+		/* ----------
+		 * pid is no longer running, so remove the lock file.
+		 * This is not 100% safe from concurrent access, e.g.:
+		 *
+		 *    process 1 exits and leaves stale lock file
+		 *    process 2 checks stale lock file of process 1
+		 *    process 3 checks stale lock file of process 1
+		 *    process 2 remove the lock file of process 1
+		 *    process 4 creates a lock file
+		 *    process 3 remove the lock file of process 4
+		 *    process 5 creates a lock file
+		 *
+		 * The sleep(2) helps with this since it reduces the likelihood
+		 * a process that did an unlock will interfere with another unlock
+		 * process.  We could ask users to remove the lock, but that seems
+		 * even more error-prone, especially since this might happen
+		 * on server start.  Many PG tools seem to have problems with
+		 * concurrent access.
+		 * ----------
+		 */
+		unlink(pid_path);
+
+		/* Sleep to reduce the likelihood of concurrent unlink */
+		pg_usleep(2000000L);	/* 2 seconds */
+	}
+
+	/* Create our own lockfile? */
+#ifndef WIN32
+	lock_fd = open(pid_path, O_RDWR | O_CREAT | O_EXCL, pg_file_create_mode);
+#else
+	/* delete on close */
+	lock_fd = open(pid_path, O_RDWR | O_CREAT | O_EXCL | O_TEMPORARY,
+				   pg_file_create_mode);
+#endif
+
+	if (lock_fd == -1)
+	{
+		if (errno == EEXIST)
+			pg_log_error("an active process currently holds a lock on this operation, recorded in \"%s\"",
+						 KMGR_DIR_PID);
+		else
+			pg_log_error("unable to create lock file \"%s\": %m", KMGR_DIR_PID);
+		fprintf(stderr, _("Exiting with no changes made.\n"));
+		exit(1);
+	}
+
+	snprintf(lock_pid_str, sizeof(lock_pid_str), "%d\n", getpid());
+	if (write(lock_fd, lock_pid_str, strlen(lock_pid_str)) != strlen(lock_pid_str))
+	{
+		pg_log_error("could not write pid to lock file \"%s\": %m", KMGR_DIR_PID);
+		fprintf(stderr, _("Exiting with no changes made.\n"));
+		exit(1);
+	}
+}
+
+/*
+ * ----------
+ *	recover_failure
+ *
+ * A previous pg_alterckey might have failed, so it might need recovery.
+ * The normal operation is:
+ * 1.     reencrypt  LIVE_KMGR_DIR -> NEW_KMGR_DIR
+ * 2.     rename     KMGR_DIR      -> OLD_KMGR_DIR
+ * 3.     rename     NEW_KMGR_DIR  -> LIVE_KMGR_DIR
+ *        remove     OLD_KMGR_DIR
+ *
+ * There are eight possible directory configurations:
+ *
+ *                            LIVE_KMGR_DIR   NEW_KMGR_DIR    OLD_KMGR_DIR
+ *
+ * Normal:
+ * 0. normal                       X
+ * 1. remove new                   X               X
+ * 2. install new                                  X               X
+ * 3. remove old                   X                               X
+ *
+ * Abnormal:
+ *    fatal
+ *    restore old                                                  X
+ *    install new                                  X
+ *    remove old and new           X               X               X
+ *
+ * We don't handle the abnormal cases, just report an error.
+ * ----------
+ */
+static void
+recover_failure(void)
+{
+	struct stat buffer;
+	bool		is_live,
+				is_new,
+				is_old;
+
+	is_live = !stat(live_path, &buffer);
+	is_new = !stat(new_path, &buffer);
+	is_old = !stat(old_path, &buffer);
+
+	/* normal #0 */
+	if (is_live && !is_new && !is_old)
+	{
+		if (repair_mode)
+			printf("repair unnecessary\n");
+		return;
+	}
+	/* remove new #1 */
+	else if (is_live && is_new && !is_old)
+	{
+		if (!rmtree(new_path, true))
+		{
+			pg_log_error("unable to remove new directory \"%s\": %m", NEW_KMGR_DIR);
+			fprintf(stderr, _("Exiting with no changes made.\n"));
+			exit(1);
+		}
+		printf(_("removed files created during previously aborted alter operation\n"));
+		return;
+	}
+	/* install new #2 */
+	else if (!is_live && is_new && is_old)
+	{
+		if (rename(new_path, live_path) != 0)
+		{
+			pg_log_error("unable to rename directory \"%s\" to \"%s\": %m",
+						 NEW_KMGR_DIR, LIVE_KMGR_DIR);
+			fprintf(stderr, _("Exiting with no changes made.\n"));
+			exit(1);
+		}
+		printf(_("Installed new cluster password supplied in previous alter operation\n"));
+		return;
+	}
+	/* remove old #3 */
+	else if (is_live && !is_new && is_old)
+	{
+		if (!rmtree(old_path, true))
+		{
+			pg_log_error("unable to remove old directory \"%s\": %m", OLD_KMGR_DIR);
+			fprintf(stderr, _("Exiting with no changes made.\n"));
+			exit(1);
+		}
+		printf(_("Removed old files invalidated during previous alter operation\n"));
+		return;
+	}
+	else
+	{
+		pg_log_error("cluster file encryption directory \"%s\" is in an abnormal state and cannot be processed",
+					 KMGR_DIR);
+		fprintf(stderr, _("Exiting with no changes made.\n"));
+		exit(1);
+	}
+}
+
+/* Retrieve old and new cluster keys */
+void
+retrieve_cluster_keys(void)
+{
+	int			cluster_key_len;
+	char		cluster_key_hex[ALLOC_KMGR_CLUSTER_KEY_MAX_LEN];
+
+	/*
+	 * If we have been asked to pass an open file descriptor to the user
+	 * terminal to the commands, set one up.
+	 */
+	if (pass_terminal_fd)
+	{
+#ifndef WIN32
+		terminal_fd = open("/dev/tty", O_RDWR, 0);
+#else
+		terminal_fd = open("CONOUT$", O_RDWR, 0);
+#endif
+		if (terminal_fd < 0)
+		{
+			pg_log_error(_("%s: could not open terminal: %s\n"),
+						 progname, strerror(errno));
+			exit(1);
+		}
+	}
+
+	/* Get old key encryption key from the cluster key command */
+	cluster_key_len = kmgr_run_cluster_key_command(old_cluster_key_cmd,
+												   (char *) cluster_key_hex,
+												   ALLOC_KMGR_CLUSTER_KEY_MAX_LEN,
+												   live_path, terminal_fd);
+	if (hex_decode(cluster_key_hex, cluster_key_len,
+					  (char *) old_cluster_key) !=
+		KMGR_CLUSTER_KEY_LEN(ControlFile->file_encryption_method))
+	{
+		pg_log_error("cluster key must be at %d hex bytes", KMGR_CLUSTER_KEY_LEN(ControlFile->file_encryption_method) * 2);
+		bzero_keys_and_exit(ERROR_EXIT);
+	}
+
+	/*
+	 * Create new key directory here in case the new cluster key command needs
+	 * it to exist.
+	 */
+	if (mkdir(new_path, pg_dir_create_mode) != 0)
+	{
+		pg_log_error("unable to create new cluster key directory \"%s\": %m", NEW_KMGR_DIR);
+		bzero_keys_and_exit(ERROR_EXIT);
+	}
+
+	/* Get new key */
+	cluster_key_len = kmgr_run_cluster_key_command(new_cluster_key_cmd,
+												   (char *) cluster_key_hex,
+												   ALLOC_KMGR_CLUSTER_KEY_MAX_LEN,
+												   new_path, terminal_fd);
+	if (hex_decode(cluster_key_hex, cluster_key_len,
+					  (char *) new_cluster_key) !=
+		KMGR_CLUSTER_KEY_LEN(ControlFile->file_encryption_method))
+	{
+		pg_log_error("cluster key must be at %d hex bytes", KMGR_CLUSTER_KEY_LEN(ControlFile->file_encryption_method) * 2);
+		bzero_keys_and_exit(ERROR_EXIT);
+	}
+
+	if (pass_terminal_fd)
+		close(terminal_fd);
+
+	/* output newline */
+	puts("");
+
+	if (memcmp(old_cluster_key, new_cluster_key, KMGR_CLUSTER_KEY_LEN(ControlFile->file_encryption_method)) == 0)
+	{
+		pg_log_error("cluster keys are identical, exiting\n");
+		bzero_keys_and_exit(RMDIR_EXIT);
+	}
+}
+
+/* Decrypt old keys encrypted with old pass phrase and reencrypt with new one */
+void
+reencrypt_data_keys(void)
+{
+	PgCipherCtx *old_ctx,
+			   *new_ctx;
+
+	old_ctx = pg_cipher_ctx_create(PG_CIPHER_AES_KWP,
+								   (unsigned char *) old_cluster_key,
+								   KMGR_KEK_KEY_LEN, false);
+	if (!old_ctx)
+		pg_log_error("could not initialize encryption context");
+
+	new_ctx = pg_cipher_ctx_create(PG_CIPHER_AES_KWP,
+								   (unsigned char *) new_cluster_key,
+								   KMGR_KEK_KEY_LEN, true);
+	if (!new_ctx)
+		pg_log_error("could not initialize encryption context");
+
+	for (int id = 0; id < KMGR_NUM_DATA_KEYS; id++)
+	{
+		char		src_path[MAXPGPATH],
+					dst_path[MAXPGPATH];
+		int			src_fd,
+					dst_fd;
+		int			len;
+		struct stat st;
+
+		CryptoKeyFilePath(src_path, live_path, id);
+		CryptoKeyFilePath(dst_path, new_path, id);
+
+		if ((src_fd = open(src_path, O_RDONLY | PG_BINARY, 0)) < 0)
+		{
+			pg_log_error("could not open file \"%s\": %m", src_path);
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		if ((dst_fd = open(dst_path, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY,
+						   pg_file_create_mode)) < 0)
+		{
+			pg_log_error("could not open file \"%s\": %m", dst_path);
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		if (fstat(src_fd, &st))
+		{
+			pg_log_error("could not stat file \"%s\": %m", src_path);
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		in_klen = st.st_size;
+
+		if (in_klen > MAX_WRAPPED_KEY_LENGTH)
+		{
+			pg_log_error("invalid wrapped key length (%d) for file \"%s\"", in_klen, src_path);
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		/* Read the source key */
+		len = read(src_fd, in_key, in_klen);
+		if (len != in_klen)
+		{
+			if (len < 0)
+				pg_log_error("could read file \"%s\": %m", src_path);
+			else
+				pg_log_error("could read file \"%s\": read %d of %u",
+							 src_path, len, in_klen);
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		/* decrypt with old key */
+		if (!kmgr_unwrap_data_key(old_ctx, in_key, in_klen, &data_key))
+		{
+			pg_log_error("incorrect old key specified");
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		if (KMGR_MAX_KEY_LEN_BYTES + pg_cipher_blocksize(new_ctx) > MAX_WRAPPED_KEY_LENGTH)
+		{
+			pg_log_error("invalid max wrapped key length");
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		/* encrypt with new key */
+		if (!kmgr_wrap_data_key(new_ctx, &data_key, out_key, &out_klen))
+		{
+			pg_log_error("could not encrypt new key");
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		/* Write to the dest key */
+		len = write(dst_fd, out_key, out_klen);
+		if (len != out_klen)
+		{
+			pg_log_error("could not write fie \"%s\"", dst_path);
+			bzero_keys_and_exit(RMDIR_EXIT);
+		}
+
+		close(src_fd);
+		close(dst_fd);
+	}
+
+	/* The cluster key is correct, free the cipher context */
+	pg_cipher_ctx_free(old_ctx);
+	pg_cipher_ctx_free(new_ctx);
+}
+
+/* Install new keys */
+void
+install_new_keys(void)
+{
+	/*
+	 * Issue fsync's so key rotation is less likely to be left in an
+	 * inconsistent state in case of a crash during this operation.
+	 */
+	
+	if (rename(live_path, old_path) != 0)
+	{
+		pg_log_error("unable to rename directory \"%s\" to \"%s\": %m",
+					 LIVE_KMGR_DIR, OLD_KMGR_DIR);
+		bzero_keys_and_exit(RMDIR_EXIT);
+	}
+	sync_dir_recurse(top_path, DATA_DIR_SYNC_METHOD_FSYNC);
+
+	if (rename(new_path, live_path) != 0)
+	{
+		pg_log_error("unable to rename directory \"%s\" to \"%s\": %m",
+					 NEW_KMGR_DIR, LIVE_KMGR_DIR);
+		bzero_keys_and_exit(REPAIR_EXIT);
+	}
+	sync_dir_recurse(top_path, DATA_DIR_SYNC_METHOD_FSYNC);
+
+	if (!rmtree(old_path, true))
+	{
+		pg_log_error("unable to remove old directory \"%s\": %m", OLD_KMGR_DIR);
+		bzero_keys_and_exit(REPAIR_EXIT);
+	}
+	sync_dir_recurse(top_path, DATA_DIR_SYNC_METHOD_FSYNC);
+}
+
+/* Erase memory and exit */
+void
+bzero_keys_and_exit(exit_action action)
+{
+	explicit_bzero(old_cluster_key, sizeof(old_cluster_key));
+	explicit_bzero(new_cluster_key, sizeof(new_cluster_key));
+
+	explicit_bzero(in_key, sizeof(in_key));
+	explicit_bzero(&data_key, sizeof(data_key));
+	explicit_bzero(out_key, sizeof(out_key));
+
+	if (action == RMDIR_EXIT)
+	{
+		if (!rmtree(new_path, true))
+			pg_log_error("unable to remove new directory \"%s\": %m", NEW_KMGR_DIR);
+		printf("Re-running pg_alterckey to repair might be needed before the next server start\n");
+		exit(1);
+	}
+	else if (action == REPAIR_EXIT)
+	{
+		unlink(pid_path);
+		printf("Re-running pg_alterckey to repair might be needed before the next server start\n");
+	}
+
+	/* return 0 or 1 */
+	exit(action != SUCCESS_EXIT);
+}
+
+/*
+ * HEX
+ */
+
+static const int8 hexlookup[128] = {
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
+	-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+	-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+};
+
+static inline char
+get_hex(const char *cp)
+{
+	unsigned char c = (unsigned char) *cp;
+	int			res = -1;
+
+	if (c < 127)
+		res = hexlookup[c];
+
+	if (res < 0)
+		pg_fatal("invalid hexadecimal digit: \"%s\"",cp);
+
+	return (char) res;
+}
+
+static uint64
+hex_decode(const char *src, size_t len, char *dst)
+{
+	const char *s,
+			   *srcend;
+	char		v1,
+				v2,
+			   *p;
+
+	srcend = src + len;
+	s = src;
+	p = dst;
+	while (s < srcend)
+	{
+		if (*s == ' ' || *s == '\n' || *s == '\t' || *s == '\r')
+		{
+			s++;
+			continue;
+		}
+		v1 = get_hex(s) << 4;
+		s++;
+		if (s >= srcend)
+			pg_fatal("invalid hexadecimal data: odd number of digits");
+
+		v2 = get_hex(s);
+		s++;
+		*p++ = v1 | v2;
+	}
+
+	return p - dst;
+}
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 93e0837947..3db9d97cec 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -25,6 +25,7 @@
 #include "access/xlog_internal.h"
 #include "catalog/pg_control.h"
 #include "common/controldata_utils.h"
+#include "common/kmgr_utils.h"
 #include "common/logging.h"
 #include "getopt_long.h"
 #include "pg_getopt.h"
@@ -327,5 +328,7 @@ main(int argc, char *argv[])
 		   ControlFile->data_checksum_version);
 	printf(_("Mock authentication nonce:            %s\n"),
 		   mock_auth_nonce_str);
+	printf(_("File encryption method:               %s\n"),
+		   encryption_methods[ControlFile->file_encryption_method].name);
 	return 0;
 }
diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c
index 3b145bd838..ca189d6ac2 100644
--- a/src/bin/pg_ctl/pg_ctl.c
+++ b/src/bin/pg_ctl/pg_ctl.c
@@ -74,6 +74,7 @@ typedef enum
 static bool do_wait = true;
 static int	wait_seconds = DEFAULT_WAIT;
 static bool wait_seconds_arg = false;
+static bool pass_terminal_fd = false;
 static bool silent_mode = false;
 static ShutdownMode shutdown_mode = FAST_MODE;
 static int	sig = SIGINT;		/* default */
@@ -438,7 +439,7 @@ free_readfile(char **optlines)
 static pid_t
 start_postmaster(void)
 {
-	char	   *cmd;
+	char	   *cmd, *term_fd_opt = NULL;
 
 #ifndef WIN32
 	pid_t		pm_pid;
@@ -466,6 +467,19 @@ start_postmaster(void)
 
 	/* fork succeeded, in child */
 
+	if (pass_terminal_fd)
+	{
+		int terminal_fd = open("/dev/tty", O_RDWR, 0);
+
+		if (terminal_fd < 0)
+		{
+			write_stderr(_("%s: could not open terminal: %s\n"),
+						 progname, strerror(errno));
+			exit(1);
+		}
+		term_fd_opt = psprintf(" -R %d", terminal_fd);
+	}
+
 	/*
 	 * If possible, detach the postmaster process from the launching process
 	 * group and make it a group leader, so that it doesn't get signaled along
@@ -486,12 +500,14 @@ start_postmaster(void)
 	 * has the same PID as the current child process.
 	 */
 	if (log_file != NULL)
-		cmd = psprintf("exec \"%s\" %s%s < \"%s\" >> \"%s\" 2>&1",
+		cmd = psprintf("exec \"%s\" %s%s%s < \"%s\" >> \"%s\" 2>&1",
 					   exec_path, pgdata_opt, post_opts,
+					   term_fd_opt ? term_fd_opt : "",
 					   DEVNULL, log_file);
 	else
-		cmd = psprintf("exec \"%s\" %s%s < \"%s\" 2>&1",
-					   exec_path, pgdata_opt, post_opts, DEVNULL);
+		cmd = psprintf("exec \"%s\" %s%s%s < \"%s\" 2>&1",
+					   exec_path, pgdata_opt, post_opts,
+					   term_fd_opt ? term_fd_opt : "", DEVNULL);
 
 	(void) execl("/bin/sh", "/bin/sh", "-c", cmd, (char *) NULL);
 
@@ -512,6 +528,21 @@ start_postmaster(void)
 	PROCESS_INFORMATION pi;
 	const char *comspec;
 
+	if (pass_terminal_fd)
+	{
+		/*  Hopefully we can read and write CONOUT, see simple_prompt() XXX */
+		/*  Do CreateRestrictedProcess() children even inherit open file descriptors? XXX */
+		int terminal_fd = open("CONOUT$", O_RDWR, 0);
+
+		if (terminal_fd < 0)
+		{
+			write_stderr(_("%s: could not open terminal: %s\n"),
+						 progname, strerror(errno));
+			exit(1);
+		}
+		term_fd_opt = psprintf(" -R %d", terminal_fd);
+	}
+
 	/* Find CMD.EXE location using COMSPEC, if it's set */
 	comspec = getenv("COMSPEC");
 	if (comspec == NULL)
@@ -552,12 +583,14 @@ start_postmaster(void)
 		else
 			close(fd);
 
-		cmd = psprintf("\"%s\" /C \"\"%s\" %s%s < \"%s\" >> \"%s\" 2>&1\"",
-					   comspec, exec_path, pgdata_opt, post_opts, DEVNULL, log_file);
+		cmd = psprintf("\"%s\" /C \"\"%s\" %s%s%s < \"%s\" >> \"%s\" 2>&1\"",
+					   comspec, exec_path, pgdata_opt, post_opts,
+					   term_fd_opt ? term_fd_opt : "", DEVNULL, log_file);
 	}
 	else
-		cmd = psprintf("\"%s\" /C \"\"%s\" %s%s < \"%s\" 2>&1\"",
-					   comspec, exec_path, pgdata_opt, post_opts, DEVNULL);
+		cmd = psprintf("\"%s\" /C \"\"%s\" %s%s%s < \"%s\" 2>&1\"",
+					   comspec, exec_path, pgdata_opt, post_opts,
+					   term_fd_opt ? term_fd_opt : "", DEVNULL);
 
 	if (!CreateRestrictedProcess(cmd, &pi, false))
 	{
@@ -688,7 +721,8 @@ wait_for_postmaster_start(pid_t pm_pid, bool do_checkpoint)
 			}
 			else
 #endif
-				print_msg(".");
+				if (!pass_terminal_fd)
+					print_msg(".");
 		}
 
 		pg_usleep(USEC_PER_SEC / WAITS_PER_SEC);
@@ -2004,6 +2038,7 @@ do_help(void)
 	printf(_("  -o, --options=OPTIONS  command line options to pass to postgres\n"
 			 "                         (PostgreSQL server executable) or initdb\n"));
 	printf(_("  -p PATH-TO-POSTGRES    normally not necessary\n"));
+	printf(_("  -R, --authprompt       prompt for a passphrase or PIN\n"));
 	printf(_("\nOptions for stop or restart:\n"));
 	printf(_("  -m, --mode=MODE        MODE can be \"smart\", \"fast\", or \"immediate\"\n"));
 
@@ -2198,6 +2233,7 @@ main(int argc, char **argv)
 		{"mode", required_argument, NULL, 'm'},
 		{"pgdata", required_argument, NULL, 'D'},
 		{"options", required_argument, NULL, 'o'},
+		{"authprompt", no_argument, NULL, 'R'},
 		{"silent", no_argument, NULL, 's'},
 		{"timeout", required_argument, NULL, 't'},
 		{"core-files", no_argument, NULL, 'c'},
@@ -2260,7 +2296,7 @@ main(int argc, char **argv)
 		wait_seconds = atoi(env_wait);
 
 	/* process command-line options */
-	while ((c = getopt_long(argc, argv, "cD:e:l:m:N:o:p:P:sS:t:U:wW",
+	while ((c = getopt_long(argc, argv, "cD:e:l:m:N:o:p:P:RsS:t:U:wW",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
@@ -2311,6 +2347,9 @@ main(int argc, char **argv)
 			case 'P':
 				register_password = pg_strdup(optarg);
 				break;
+			case 'R':
+				pass_terminal_fd = true;
+				break;
 			case 's':
 				silent_mode = true;
 				break;
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 3ae3fc06df..ef6d50c93b 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -51,6 +51,7 @@
 #include "common/controldata_utils.h"
 #include "common/fe_memutils.h"
 #include "common/file_perm.h"
+#include "common/kmgr_utils.h"
 #include "common/logging.h"
 #include "common/restricted_token.h"
 #include "common/string.h"
@@ -777,6 +778,8 @@ PrintControlValues(bool guessed)
 		   (ControlFile.float8ByVal ? _("by value") : _("by reference")));
 	printf(_("Data page checksum version:           %u\n"),
 		   ControlFile.data_checksum_version);
+	printf(_("File encryption method:               %s\n"),
+		   encryption_methods[ControlFile.file_encryption_method].name);
 }
 
 
diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c
index ecadd69dc5..7a829284e1 100644
--- a/src/bin/pg_rewind/filemap.c
+++ b/src/bin/pg_rewind/filemap.c
@@ -29,6 +29,7 @@
 #include "catalog/pg_tablespace_d.h"
 #include "common/file_utils.h"
 #include "common/hashfn.h"
+#include "common/kmgr_utils.h"
 #include "common/string.h"
 #include "datapagemap.h"
 #include "filemap.h"
@@ -106,6 +107,13 @@ static const char *const excludeDirContents[] =
 	/* Contents removed on startup, see AsyncShmemInit(). */
 	"pg_notify",
 
+	/*
+	 * Skip cryptographic keys. It's generally not a good idea to copy the
+	 * cryptographic keys from source database because these might use
+	 * different cluster key.
+	 */
+	KMGR_DIR,
+
 	/*
 	 * Old contents are loaded for possible debugging but are not required for
 	 * normal operation, see SerialInit().
diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c
index fa52aa2c22..743ae3f5a6 100644
--- a/src/bin/pg_upgrade/check.c
+++ b/src/bin/pg_upgrade/check.c
@@ -11,6 +11,7 @@
 
 #include "catalog/pg_authid_d.h"
 #include "catalog/pg_collation.h"
+#include "common/kmgr_utils.h"
 #include "fe_utils/string_utils.h"
 #include "mb/pg_wchar.h"
 #include "pg_upgrade.h"
@@ -31,6 +32,7 @@ static void check_for_removed_data_type_usage(ClusterInfo *cluster,
 											  const char *datatype);
 static void check_for_jsonb_9_4_usage(ClusterInfo *cluster);
 static void check_for_pg_role_prefix(ClusterInfo *cluster);
+static void check_for_cluster_key_failure(ClusterInfo *cluster);
 static void check_for_new_tablespace_dir(void);
 static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster);
 static void check_new_cluster_logical_replication_slots(void);
@@ -189,6 +191,9 @@ check_and_dump_old_cluster(bool live_check)
 	if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905)
 		check_for_pg_role_prefix(&old_cluster);
 
+	if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1400)
+		check_for_cluster_key_failure(&old_cluster);
+
 	if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 &&
 		old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER)
 		check_for_jsonb_9_4_usage(&old_cluster);
@@ -218,6 +223,9 @@ check_new_cluster(void)
 
 	check_loadable_libraries();
 
+	if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1400)
+		check_for_cluster_key_failure(&new_cluster);
+
 	switch (user_opts.transfer_mode)
 	{
 		case TRANSFER_MODE_CLONE:
@@ -1613,3 +1621,29 @@ check_old_cluster_for_valid_slots(bool live_check)
 
 	check_ok();
 }
+
+
+/*
+ * check_for_cluster_key_failure()
+ *
+ *	Make sure there was no unrepaired pg_alterckey failure
+ */
+static void
+check_for_cluster_key_failure(ClusterInfo *cluster)
+{
+	struct stat buffer;
+
+	if (stat (KMGR_DIR_PID, &buffer) == 0)
+	{
+		if (cluster == &old_cluster)
+			pg_fatal("The source cluster had a pg_alterckey failure that needs repair or\n"
+					 "pg_alterckey is running.  Run pg_alterckey --repair or wait for it\n"
+					 "to complete.");
+		else
+			pg_fatal("The target cluster had a pg_alterckey failure that needs repair or\n"
+					 "pg_alterckey is running.  Run pg_alterckey --repair or wait for it\n"
+					 "to complete.");
+	}
+
+	check_ok();
+}
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 4beb65ab22..ae3ddd7f6e 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -9,12 +9,18 @@
 
 #include "postgres_fe.h"
 
+#include <dirent.h>
 #include <ctype.h>
 
 #include "pg_upgrade.h"
 #include "common/string.h"
 
 
+#include "access/xlog_internal.h"
+#include "common/controldata_utils.h"
+#include "common/file_utils.h"
+#include "common/kmgr_utils.h"
+
 /*
  * get_control_data()
  *
@@ -62,6 +68,7 @@ get_control_data(ClusterInfo *cluster, bool live_check)
 	bool		got_date_is_int = false;
 	bool		got_data_checksum_version = false;
 	bool		got_cluster_state = false;
+	int			got_file_encryption_method = false;
 	char	   *lc_collate = NULL;
 	char	   *lc_ctype = NULL;
 	char	   *lc_monetary = NULL;
@@ -207,6 +214,13 @@ get_control_data(ClusterInfo *cluster, bool live_check)
 		got_data_checksum_version = true;
 	}
 
+	/* Only in <= 16 */
+	if (GET_MAJOR_VERSION(cluster->major_version) <= 1600)
+	{
+		cluster->controldata.file_encryption_method = DISABLED_ENCRYPTION_METHOD;
+		got_file_encryption_method = true;
+	}
+
 	/* we have the result of cmd in "output". so parse it line by line now */
 	while (fgets(bufin, sizeof(bufin), output))
 	{
@@ -502,6 +516,18 @@ get_control_data(ClusterInfo *cluster, bool live_check)
 			cluster->controldata.data_checksum_version = str2uint(p);
 			got_data_checksum_version = true;
 		}
+		else if ((p = strstr(bufin, "File encryption method:")) != NULL)
+		{
+			p = strchr(p, ':');
+
+			if (p == NULL || strlen(p) <= 1)
+				pg_fatal("%d: controldata retrieval problem", __LINE__);
+
+			p++;				/* remove ':' char */
+			/* used later for contrib check */
+			cluster->controldata.file_encryption_method = atoi(p);
+			got_file_encryption_method = true;
+		}
 	}
 
 	rc = pclose(output);
@@ -573,7 +599,8 @@ get_control_data(ClusterInfo *cluster, bool live_check)
 		!got_index || !got_toast ||
 		(!got_large_object &&
 		 cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) ||
-		!got_date_is_int || !got_data_checksum_version)
+		!got_date_is_int || !got_data_checksum_version ||
+		!got_file_encryption_method)
 	{
 		if (cluster == &old_cluster)
 			pg_log(PG_REPORT,
@@ -642,6 +669,10 @@ get_control_data(ClusterInfo *cluster, bool live_check)
 		if (!got_data_checksum_version)
 			pg_log(PG_REPORT, "  data checksum version");
 
+		/* value added in Postgres 16 */
+		if (!got_file_encryption_method)
+			pg_log(PG_REPORT, "  file encryption method");
+
 		pg_fatal("Cannot continue without required control information, terminating");
 	}
 }
@@ -706,6 +737,14 @@ check_control_data(ControlData *oldctrl,
 		pg_fatal("old cluster uses data checksums but the new one does not");
 	else if (oldctrl->data_checksum_version != newctrl->data_checksum_version)
 		pg_fatal("old and new cluster pg_controldata checksum versions do not match");
+
+	/*
+	 * We cannot upgrade if the old cluster file encryption method
+	 * doesn't match the new one.
+	 */
+	if (oldctrl->file_encryption_method != newctrl->file_encryption_method)
+		pg_fatal("old and new clusters use different file encryption methods or\n"
+				 "one cluster uses encryption and the other does not");
 }
 
 
diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c
index d173602882..d0aa2434f9 100644
--- a/src/bin/pg_upgrade/file.c
+++ b/src/bin/pg_upgrade/file.c
@@ -11,6 +11,7 @@
 
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <dirent.h>
 #ifdef HAVE_COPYFILE_H
 #include <copyfile.h>
 #endif
@@ -21,6 +22,7 @@
 
 #include "access/visibilitymapdefs.h"
 #include "common/file_perm.h"
+#include "common/file_utils.h"
 #include "pg_upgrade.h"
 #include "storage/bufpage.h"
 #include "storage/checksum.h"
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index b9d900d0db..d68616a657 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -53,6 +53,7 @@ parseCommandLine(int argc, char *argv[])
 		{"check", no_argument, NULL, 'c'},
 		{"link", no_argument, NULL, 'k'},
 		{"retain", no_argument, NULL, 'r'},
+		{"authprompt", no_argument, NULL, 'R'},
 		{"jobs", required_argument, NULL, 'j'},
 		{"socketdir", required_argument, NULL, 's'},
 		{"verbose", no_argument, NULL, 'v'},
@@ -103,7 +104,7 @@ parseCommandLine(int argc, char *argv[])
 	if (os_user_effective_id == 0)
 		pg_fatal("%s: cannot be run as root", os_info.progname);
 
-	while ((option = getopt_long(argc, argv, "b:B:cd:D:j:kNo:O:p:P:rs:U:v",
+	while ((option = getopt_long(argc, argv, "d:D:b:B:cj:kNo:O:p:P:rRs:U:v",
 								 long_options, &optindex)) != -1)
 	{
 		switch (option)
@@ -180,6 +181,10 @@ parseCommandLine(int argc, char *argv[])
 				log_opts.retain = true;
 				break;
 
+			case 'R':
+				user_opts.pass_terminal_fd = true;
+				break;
+
 			case 's':
 				user_opts.socketdir = pg_strdup(optarg);
 				break;
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index a710f325de..a47fd18741 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -12,6 +12,7 @@
 
 #include "common/relpath.h"
 #include "libpq-fe.h"
+#include "common/kmgr_utils.h"
 
 /* For now, pg_upgrade does not use common/logging.c; use our own pg_fatal */
 #undef pg_fatal
@@ -244,6 +245,7 @@ typedef struct
 	bool		date_is_int;
 	bool		float8_pass_by_value;
 	uint32		data_checksum_version;
+	int			file_encryption_method;
 } ControlData;
 
 /*
@@ -324,6 +326,8 @@ typedef struct
 	int			jobs;			/* number of processes/threads to use */
 	char	   *socketdir;		/* directory to use for Unix sockets */
 	char	   *sync_method;
+	bool		ind_coll_unknown;	/* mark unknown index collation versions */
+	bool		pass_terminal_fd; /* pass -R to pg_ctl? */
 } UserOpts;
 
 typedef struct
diff --git a/src/bin/pg_upgrade/server.c b/src/bin/pg_upgrade/server.c
index d7f6c268ef..a896b3d2a2 100644
--- a/src/bin/pg_upgrade/server.c
+++ b/src/bin/pg_upgrade/server.c
@@ -254,7 +254,7 @@ start_postmaster(ClusterInfo *cluster, bool report_and_exit_on_error)
 
 	/* Use -b to disable autovacuum. */
 	snprintf(cmd, sizeof(cmd),
-			 "\"%s/pg_ctl\" -w -l \"%s/%s\" -D \"%s\" -o \"-p %d -b%s %s%s\" start",
+			 "\"%s/pg_ctl\" -w -l \"%s/%s\" -D \"%s\" -o \"-p %d%s -b %s%s\" start",
 			 cluster->bindir,
 			 log_opts.logdir,
 			 SERVER_LOG_FILE, cluster->pgconfig, cluster->port,
-- 
2.40.1

