Received: from malur.postgresql.org ([217.196.149.56]) by arkaria.postgresql.org with esmtps (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (Exim 4.94.2) (envelope-from ) id 1sXo5S-006W59-Jc for pgsql-hackers@arkaria.postgresql.org; Sat, 27 Jul 2024 20:32:18 +0000 Received: from localhost ([127.0.0.1] helo=malur.postgresql.org) by malur.postgresql.org with esmtp (Exim 4.94.2) (envelope-from ) id 1sXo5Q-007FKk-MT for pgsql-hackers@arkaria.postgresql.org; Sat, 27 Jul 2024 20:32:16 +0000 Received: from makus.postgresql.org ([2001:4800:3e1:1::229]) by malur.postgresql.org with esmtps (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (Exim 4.94.2) (envelope-from ) id 1sXo5Q-007FKc-3m for pgsql-hackers@lists.postgresql.org; Sat, 27 Jul 2024 20:32:16 +0000 Received: from mail-il1-x136.google.com ([2607:f8b0:4864:20::136]) by makus.postgresql.org with esmtps (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 (Exim 4.94.2) (envelope-from ) id 1sXo5M-001lda-M9 for pgsql-hackers@postgresql.org; Sat, 27 Jul 2024 20:32:14 +0000 Received: by mail-il1-x136.google.com with SMTP id e9e14a558f8ab-39ae90e3263so4696255ab.1 for ; Sat, 27 Jul 2024 13:32:12 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1722112331; x=1722717131; darn=postgresql.org; h=in-reply-to:content-disposition:mime-version:references:message-id :subject:cc:to:from:date:from:to:cc:subject:date:message-id:reply-to; bh=qJVWpZfgdV5YFTI71dw0lKFx3kAB7Xs12hf7OqZU5gg=; b=VHpNjYLXfvmixDIFnGA9ydwSqzvvGV5MyEgxpoiVFxNfL/e7/pdJBRWDkB2NosVP33 O8qbHYTTz0qGGm8dV51glbSllg/M/ZWSdZbqs5IhdDaBafEfRZRVIIBKK/COUaiF0wcP KYdAzOmgKnh+FNRh2E2F9lCDc40b9QlahlkxSsvrEB7mttg+NzQrJIBLjgjPZK/0MJbA pulE4z+alNl+wqz/Q/y64BKsEEVkdkRthndoZ7oocmr0jqfwMxNx8Id2M8l4BYpLJLSQ 1foi46PB6EVfPp+yfWzBjqI5j3K+h1EKLahB0GJ+A7H+X228mIT9QaoNlFc0+rxSwHOM tuFw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1722112331; x=1722717131; h=in-reply-to:content-disposition:mime-version:references:message-id :subject:cc:to:from:date:x-gm-message-state:from:to:cc:subject:date :message-id:reply-to; bh=qJVWpZfgdV5YFTI71dw0lKFx3kAB7Xs12hf7OqZU5gg=; b=RiUfpYsbLTRovFcFij7r/O+/Ro/RAXb6jkaBHO3crkOosXnPnVGyk+OzTzciQYy7v5 Sx/Iay1sBGuQhzvhGAqSzm4QFZDYHixW0S+dBSXsM3SNPL3ti15v4G5/ssvWbnhGrzMU HndFJWZIWAlq8oUYUx0QpAEojIfC21iY8HQkHbimb0+qM4gYwNFa39SoCkhua1p3j1W3 3s7cF8cupHj97Bzqgd0jQHxNZL6ddKsohL5bZDZzaUW/ac16Wj9xK37dlJrHAit14d8y Zjz9NBzBAh7XSZe+JRQ0wrgq48syzR+O8WOOg1nnB5ZnUb6L1G5l1VJNfe//HrvrZkP8 0P1g== X-Forwarded-Encrypted: i=1; AJvYcCWyuBI8W+8XgDbIzJQbTKqD+mjn15KwB9YBX0DKxHcExwWjGT5h9AZ/4T4LdFjrPMPXTHMOY64Yjk9CUyMQ/8B8Wp8VzMuAenuNTQF1 X-Gm-Message-State: AOJu0YxhZS0zJLDXS43hLCV7GKDWSHuWfcHQbfGGPnZ5ZWNvVo8+77RF oGpIoyNTO5A03IaXQSDmbpebjTZ7YBLEvDGRyRBPo/zoZ9khibx9 X-Google-Smtp-Source: AGHT+IEn9TE2KWg14iWefJ9JhC76jgo76IgQqzoDWisORsMx07iSCjn8rHSkL1SsPoWKPBbud/+5NQ== X-Received: by 2002:a05:6e02:15c3:b0:376:d20:1470 with SMTP id e9e14a558f8ab-39aebda2886mr22706905ab.12.1722112331349; Sat, 27 Jul 2024 13:32:11 -0700 (PDT) Received: from nathan (162-195-168-172.lightspeed.stlsmo.sbcglobal.net. [162.195.168.172]) by smtp.gmail.com with ESMTPSA id 8926c6da1cb9f-4c29faf8633sm1458034173.79.2024.07.27.13.32.10 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sat, 27 Jul 2024 13:32:10 -0700 (PDT) Date: Sat, 27 Jul 2024 15:32:09 -0500 From: Nathan Bossart To: Andres Freund Cc: "Imseih (AWS), Sami" , Justin Pryzby , "pgsql-hackers@postgresql.org" Subject: Re: allow changing autovacuum_max_workers without restarting Message-ID: References: <20240603232427.pjiyqggsr7duccr5@awork3.anarazel.de> <20240618204334.i5ar2fie4vbb3fhm@awork3.anarazel.de> <20240618213331.ef2spg3nasksisbi@awork3.anarazel.de> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="s+zNQ+2ClMpNw9Ap" Content-Disposition: inline In-Reply-To: List-Id: List-Help: List-Subscribe: List-Post: List-Owner: List-Archive: Archived-At: Precedence: bulk --s+zNQ+2ClMpNw9Ap Content-Type: text/plain; charset=us-ascii Content-Disposition: inline rebased -- nathan --s+zNQ+2ClMpNw9Ap Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="v9-0001-allow-changing-autovacuum_max_workers-without-res.patch" From 61513f744012c2b9b59085ce8c4a960da9e56ee7 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Sat, 22 Jun 2024 15:05:44 -0500 Subject: [PATCH v9 1/1] allow changing autovacuum_max_workers without restarting --- doc/src/sgml/config.sgml | 28 ++++++- doc/src/sgml/runtime.sgml | 4 +- src/backend/access/transam/xlog.c | 2 +- src/backend/postmaster/autovacuum.c | 76 +++++++++++++++---- src/backend/postmaster/postmaster.c | 2 +- src/backend/storage/lmgr/proc.c | 6 +- src/backend/utils/init/postinit.c | 6 +- src/backend/utils/misc/guc_tables.c | 11 ++- src/backend/utils/misc/postgresql.conf.sample | 3 +- src/include/postmaster/autovacuum.h | 1 + src/test/perl/PostgreSQL/Test/Cluster.pm | 1 + 11 files changed, 111 insertions(+), 29 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 57cd7bb972..7fc270c0ae 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8552,6 +8552,25 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + autovacuum_worker_slots (integer) + + autovacuum_worker_slots configuration parameter + + + + + Specifies the number of backend slots to reserve for autovacuum worker + processes. The default is 16. This parameter can only be set at server + start. + + + When changing this value, consider also adjusting + . + + + + autovacuum_max_workers (integer) @@ -8562,7 +8581,14 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Specifies the maximum number of autovacuum processes (other than the autovacuum launcher) that may be running at any one time. The default - is three. This parameter can only be set at server start. + is three. This parameter can only be set in the + postgresql.conf file or on the server command line. + + + Note that a setting for this value which is higher than + will have no effect, + since autovacuum workers are taken from the pool of slots established + by that setting. diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml index 2c4d5ef640..a1f43556ac 100644 --- a/doc/src/sgml/runtime.sgml +++ b/doc/src/sgml/runtime.sgml @@ -838,7 +838,7 @@ psql: error: connection to server on socket "/tmp/.s.PGSQL.5432" failed: No such When using System V semaphores, PostgreSQL uses one semaphore per allowed connection (), allowed autovacuum worker process - (), allowed WAL sender process + (), allowed WAL sender process (), allowed background process (), etc., in sets of 16. The runtime-computed parameter @@ -891,7 +891,7 @@ $ postgres -D $PGDATA -C num_os_semaphores When using POSIX semaphores, the number of semaphores needed is the same as for System V, that is one semaphore per allowed connection (), allowed autovacuum worker process - (), allowed WAL sender process + (), allowed WAL sender process (), allowed background process (), etc. On the platforms where this option is preferred, there is no specific diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f86f4b5c4b..64c1edb798 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5373,7 +5373,7 @@ CheckRequiredParameterValues(void) */ if (ArchiveRecoveryRequested && EnableHotStandby) { - /* We ignore autovacuum_max_workers when we make this test. */ + /* We ignore autovacuum_worker_slots when we make this test. */ RecoveryRequiresIntParameter("max_connections", MaxConnections, ControlFile->MaxConnections); diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 4e4a0ccbef..937cd940aa 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -115,6 +115,7 @@ * GUC parameters */ bool autovacuum_start_daemon = false; +int autovacuum_worker_slots; int autovacuum_max_workers; int autovacuum_work_mem = -1; int autovacuum_naptime; @@ -210,7 +211,7 @@ typedef struct autovac_table /*------------- * This struct holds information about a single worker's whereabouts. We keep * an array of these in shared memory, sized according to - * autovacuum_max_workers. + * autovacuum_worker_slots. * * wi_links entry into free list or running list * wi_dboid OID of the database this worker is supposed to work on @@ -290,7 +291,7 @@ typedef struct { sig_atomic_t av_signal[AutoVacNumSignals]; pid_t av_launcherpid; - dlist_head av_freeWorkers; + dclist_head av_freeWorkers; dlist_head av_runningWorkers; WorkerInfo av_startingWorker; AutoVacuumWorkItem av_workItems[NUM_WORKITEMS]; @@ -348,6 +349,8 @@ static void autovac_report_activity(autovac_table *tab); static void autovac_report_workitem(AutoVacuumWorkItem *workitem, const char *nspname, const char *relname); static void avl_sigusr2_handler(SIGNAL_ARGS); +static bool av_worker_available(void); +static void CheckAutovacuumWorkerGUCs(void); @@ -424,6 +427,12 @@ AutoVacLauncherMain(char *startup_data, size_t startup_data_len) ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(AutovacMemCxt); + /* + * Emit a WARNING if autovacuum_worker_slots < autovacuum_max_workers. We + * do this on startup and on subsequent configuration reloads as needed. + */ + CheckAutovacuumWorkerGUCs(); + /* * If an exception is encountered, processing resumes here. * @@ -576,8 +585,7 @@ AutoVacLauncherMain(char *startup_data, size_t startup_data_len) * wakening conditions. */ - launcher_determine_sleep(!dlist_is_empty(&AutoVacuumShmem->av_freeWorkers), - false, &nap); + launcher_determine_sleep(av_worker_available(), false, &nap); /* * Wait until naptime expires or we get some type of signal (all the @@ -637,7 +645,7 @@ AutoVacLauncherMain(char *startup_data, size_t startup_data_len) current_time = GetCurrentTimestamp(); LWLockAcquire(AutovacuumLock, LW_SHARED); - can_launch = !dlist_is_empty(&AutoVacuumShmem->av_freeWorkers); + can_launch = av_worker_available(); if (AutoVacuumShmem->av_startingWorker != NULL) { @@ -680,8 +688,8 @@ AutoVacLauncherMain(char *startup_data, size_t startup_data_len) worker->wi_sharedrel = false; worker->wi_proc = NULL; worker->wi_launchtime = 0; - dlist_push_head(&AutoVacuumShmem->av_freeWorkers, - &worker->wi_links); + dclist_push_head(&AutoVacuumShmem->av_freeWorkers, + &worker->wi_links); AutoVacuumShmem->av_startingWorker = NULL; ereport(WARNING, errmsg("autovacuum worker took too long to start; canceled")); @@ -746,6 +754,8 @@ HandleAutoVacLauncherInterrupts(void) if (ConfigReloadPending) { + int autovacuum_max_workers_prev = autovacuum_max_workers; + ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); @@ -753,6 +763,14 @@ HandleAutoVacLauncherInterrupts(void) if (!AutoVacuumingActive()) AutoVacLauncherShutdown(); + /* + * If autovacuum_max_workers changed, emit a WARNING if + * autovacuum_worker_slots < autovacuum_max_workers. If it didn't + * change, skip this to avoid too many repeated log messages. + */ + if (autovacuum_max_workers_prev != autovacuum_max_workers) + CheckAutovacuumWorkerGUCs(); + /* rebuild the list in case the naptime changed */ rebuild_database_list(InvalidOid); } @@ -1088,7 +1106,7 @@ do_start_worker(void) /* return quickly when there are no free workers */ LWLockAcquire(AutovacuumLock, LW_SHARED); - if (dlist_is_empty(&AutoVacuumShmem->av_freeWorkers)) + if (!av_worker_available()) { LWLockRelease(AutovacuumLock); return InvalidOid; @@ -1241,7 +1259,7 @@ do_start_worker(void) * Get a worker entry from the freelist. We checked above, so there * really should be a free slot. */ - wptr = dlist_pop_head_node(&AutoVacuumShmem->av_freeWorkers); + wptr = dclist_pop_head_node(&AutoVacuumShmem->av_freeWorkers); worker = dlist_container(WorkerInfoData, wi_links, wptr); worker->wi_dboid = avdb->adw_datid; @@ -1610,8 +1628,8 @@ FreeWorkerInfo(int code, Datum arg) MyWorkerInfo->wi_proc = NULL; MyWorkerInfo->wi_launchtime = 0; pg_atomic_clear_flag(&MyWorkerInfo->wi_dobalance); - dlist_push_head(&AutoVacuumShmem->av_freeWorkers, - &MyWorkerInfo->wi_links); + dclist_push_head(&AutoVacuumShmem->av_freeWorkers, + &MyWorkerInfo->wi_links); /* not mine anymore */ MyWorkerInfo = NULL; @@ -3272,7 +3290,7 @@ AutoVacuumShmemSize(void) */ size = sizeof(AutoVacuumShmemStruct); size = MAXALIGN(size); - size = add_size(size, mul_size(autovacuum_max_workers, + size = add_size(size, mul_size(autovacuum_worker_slots, sizeof(WorkerInfoData))); return size; } @@ -3299,7 +3317,7 @@ AutoVacuumShmemInit(void) Assert(!found); AutoVacuumShmem->av_launcherpid = 0; - dlist_init(&AutoVacuumShmem->av_freeWorkers); + dclist_init(&AutoVacuumShmem->av_freeWorkers); dlist_init(&AutoVacuumShmem->av_runningWorkers); AutoVacuumShmem->av_startingWorker = NULL; memset(AutoVacuumShmem->av_workItems, 0, @@ -3309,10 +3327,10 @@ AutoVacuumShmemInit(void) MAXALIGN(sizeof(AutoVacuumShmemStruct))); /* initialize the WorkerInfo free list */ - for (i = 0; i < autovacuum_max_workers; i++) + for (i = 0; i < autovacuum_worker_slots; i++) { - dlist_push_head(&AutoVacuumShmem->av_freeWorkers, - &worker[i].wi_links); + dclist_push_head(&AutoVacuumShmem->av_freeWorkers, + &worker[i].wi_links); pg_atomic_init_flag(&worker[i].wi_dobalance); } @@ -3348,3 +3366,29 @@ check_autovacuum_work_mem(int *newval, void **extra, GucSource source) return true; } + +/* + * Returns whether there is a free autovacuum worker slot available. + */ +static bool +av_worker_available(void) +{ + int reserved = autovacuum_worker_slots - autovacuum_max_workers; + + return dclist_count(&AutoVacuumShmem->av_freeWorkers) > Max(0, reserved); +} + +/* + * Emit a WARNING if autovacuum_worker_slots < autovacuum_max_workers. + */ +static void +CheckAutovacuumWorkerGUCs(void) +{ + if (autovacuum_worker_slots < autovacuum_max_workers) + ereport(WARNING, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"autovacuum_max_workers\" (%d) should be less than or equal to \"autovacuum_worker_slots\" (%d)", + autovacuum_max_workers, autovacuum_worker_slots), + errdetail("The server will continue running but will only start up to \"autovacuum_worker_slots\" (%d) autovacuum workers at a given time.", + autovacuum_worker_slots))); +} diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 6f974a8d21..e4c824fcb1 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4144,7 +4144,7 @@ CreateOptsFile(int argc, char *argv[], char *fullprogname) int MaxLivePostmasterChildren(void) { - return 2 * (MaxConnections + autovacuum_max_workers + 1 + + return 2 * (MaxConnections + autovacuum_worker_slots + 1 + max_wal_senders + max_worker_processes); } diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 1b23efb26f..c20c9338ec 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -142,7 +142,7 @@ ProcGlobalSemas(void) * So, now we grab enough semaphores to support the desired max number * of backends immediately at initialization --- if the sysadmin has set * MaxConnections, max_worker_processes, max_wal_senders, or - * autovacuum_max_workers higher than his kernel will support, he'll + * autovacuum_worker_slots higher than his kernel will support, he'll * find out sooner rather than later. * * Another reason for creating semaphores here is that the semaphore @@ -242,13 +242,13 @@ InitProcGlobal(void) dlist_push_tail(&ProcGlobal->freeProcs, &proc->links); proc->procgloballist = &ProcGlobal->freeProcs; } - else if (i < MaxConnections + autovacuum_max_workers + 1) + else if (i < MaxConnections + autovacuum_worker_slots + 1) { /* PGPROC for AV launcher/worker, add to autovacFreeProcs list */ dlist_push_tail(&ProcGlobal->autovacFreeProcs, &proc->links); proc->procgloballist = &ProcGlobal->autovacFreeProcs; } - else if (i < MaxConnections + autovacuum_max_workers + 1 + max_worker_processes) + else if (i < MaxConnections + autovacuum_worker_slots + 1 + max_worker_processes) { /* PGPROC for bgworker, add to bgworkerFreeProcs list */ dlist_push_tail(&ProcGlobal->bgworkerFreeProcs, &proc->links); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 25867c8bd5..acbae29baf 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -577,15 +577,15 @@ InitializeMaxBackends(void) Assert(MaxBackends == 0); /* the extra unit accounts for the autovacuum launcher */ - MaxBackends = MaxConnections + autovacuum_max_workers + 1 + + MaxBackends = MaxConnections + autovacuum_worker_slots + 1 + max_worker_processes + max_wal_senders; if (MaxBackends > MAX_BACKENDS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("too many server processes configured"), - errdetail("\"max_connections\" (%d) plus \"autovacuum_max_workers\" (%d) plus \"max_worker_processes\" (%d) plus \"max_wal_senders\" (%d) must be less than %d.", - MaxConnections, autovacuum_max_workers, + errdetail("\"max_connections\" (%d) plus \"autovacuum_worker_slots\" (%d) plus \"max_worker_processes\" (%d) plus \"max_wal_senders\" (%d) must be less than %d.", + MaxConnections, autovacuum_worker_slots, max_worker_processes, max_wal_senders, MAX_BACKENDS))); } diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 6a623f5f34..19f6384638 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3385,7 +3385,16 @@ struct config_int ConfigureNamesInt[] = }, { /* see max_connections */ - {"autovacuum_max_workers", PGC_POSTMASTER, AUTOVACUUM, + {"autovacuum_worker_slots", PGC_POSTMASTER, AUTOVACUUM, + gettext_noop("Sets the number of backend slots to allocate for autovacuum workers."), + NULL + }, + &autovacuum_worker_slots, + 16, 1, MAX_BACKENDS, + NULL, NULL, NULL + }, + { + {"autovacuum_max_workers", PGC_SIGHUP, AUTOVACUUM, gettext_noop("Sets the maximum number of simultaneously running autovacuum worker processes."), NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 9ec9f97e92..52a4d44b59 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -658,8 +658,9 @@ #autovacuum = on # Enable autovacuum subprocess? 'on' # requires track_counts to also be on. -#autovacuum_max_workers = 3 # max number of autovacuum subprocesses +autovacuum_worker_slots = 16 # autovacuum worker slots to allocate # (change requires restart) +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses #autovacuum_naptime = 1min # time between autovacuum runs #autovacuum_vacuum_threshold = 50 # min number of row updates before # vacuum diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h index cae1e8b329..190baa699d 100644 --- a/src/include/postmaster/autovacuum.h +++ b/src/include/postmaster/autovacuum.h @@ -28,6 +28,7 @@ typedef enum /* GUC variables */ extern PGDLLIMPORT bool autovacuum_start_daemon; +extern PGDLLIMPORT int autovacuum_worker_slots; extern PGDLLIMPORT int autovacuum_max_workers; extern PGDLLIMPORT int autovacuum_work_mem; extern PGDLLIMPORT int autovacuum_naptime; diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm index 32ee98aebc..a5047038f3 100644 --- a/src/test/perl/PostgreSQL/Test/Cluster.pm +++ b/src/test/perl/PostgreSQL/Test/Cluster.pm @@ -617,6 +617,7 @@ sub init } print $conf "max_wal_senders = 10\n"; print $conf "max_replication_slots = 10\n"; + print $conf "autovacuum_worker_slots = 3\n"; print $conf "wal_log_hints = on\n"; print $conf "hot_standby = on\n"; # conservative settings to ensure we can run multiple postmasters: -- 2.39.3 (Apple Git-146) --s+zNQ+2ClMpNw9Ap--