From: CyberDem0n (@CyberDem0n) <noreply+CyberDem0n@github.com>
To: patroni/patroni <noreply+patroni-patroni@github.com>
Subject: [patroni/patroni] PR #3794: Retry watchdog activation after configuration changes
Date: Tue, 17 Nov 2026 18:31:14 +0000
Message-ID: <gh-patroni-patroni-3794@github.com> (raw)
When activation falls back to `NullWatchdog`, the configured device must be acquired again once a configuration change makes it usable. Only `keepalive()` did that, and only after `mode` was changed from `off`. But `keepalive()` is only called by the leader, and `reload_config()` of an inactive watchdog reacted only to `driver` changes. Therefore:
- If the watchdog was activated while `ttl` was less than 2x `loop_wait`, the device was never reopened after the timing was fixed. Instead, every keepalive logged `Error while sending keepalive: Setting timeout is not supported on NullWatchdog`.
- Enabling the watchdog with a reload on a replica or on a paused primary had no effect until restart. In `required` mode the node reported `watchdog_failed` and couldn't become the leader. A primary that was paused while `mode` was changed from `off` to `required` was demoted on resume and stayed a replica. In `automatic` mode the node was promoted later without a watchdog and without any warning.
- In `required` mode the HA loop gives up the leader key after a failed activation, but the watchdog stayed active. A reload that fixed the configuration was therefore deferred to a keepalive that never came, and the node stayed unhealthy until restart. E.g. after resuming a cluster in which `ttl` was set below 2x `loop_wait` during pause.
Remember that the device has to be acquired again after the timing check replaced it with `NullWatchdog`. Recreate the implementation in `activate()`, `keepalive()` and `reload_config()` of an inactive watchdog if it isn't running and that is pending, or if `mode` was changed. If activation fails, keep the watchdog active only while it is running.
Extracted from #3786.
### Reproduction
Single-node cluster on Raft with `ttl: 30`, `loop_wait: 10`, `watchdog.driver: testing`, and a FIFO reader on the device that emulates the softdog semantics (armed on open, disarmed on magic close, logs when the host would be reset).
| Scenario | master | this PR |
|---|---|---|
| `automatic`, `safety_margin: 12`, started with `ttl: 20`, then `edit-config -s ttl=30` | the device is never opened, `Error while sending keepalive` on every loop | `activated with 18 second timeout` 10 seconds later |
| pause, `mode: off` -> `automatic` and reload, resume | primary without the device ever being opened | `activated with 25 second timeout` right after resume |
| pause, `mode: off` -> `required` and reload, resume | demoted, replica with `watchdog_failed` until restart | stays primary, `activated with 25 second timeout` |
| `required`, `safety_margin: 12`: pause, `edit-config -s ttl=20`, resume, then `edit-config -s ttl=30` | demoted, replica with `watchdog_failed` until restart | demoted, then `activated with 18 second timeout` and promoted 10 seconds after `ttl=30` |
New tests: `test_reacquire_after_unsafe_timing`, `test_mode_off_after_unsafe_timing` and `test_reacquire_after_mode_change`.
inline patch 1: patroni/watchdog/base.py, tests/test_watchdog.py (+79/-4 lines) download inline-1.patchdiff --git a/patroni/watchdog/base.py b/patroni/watchdog/base.pyindex be731b98d3..5599226dbf 100644--- a/patroni/watchdog/base.py+++ b/patroni/watchdog/base.py@@ -92,6 +92,7 @@ def __init__(self, config: Config) -> None:
self.active_config: WatchdogConfig = self.config
self.lock = RLock()
self.active = False
+ self._impl_pending = False
if self.config.mode == MODE_OFF:
self.impl = NullWatchdog()
@@ -110,12 +111,15 @@ def reload_config(self, config: Config) -> None:
self._disable()
self.active_config = self.config
self.impl = NullWatchdog()
+ self._impl_pending = False
# If watchdog is not active we can apply config immediately to show any warnings early. Otherwise we need to
# delay until next time a keepalive is sent so timeout matches up with leader key update.
if not self.active:
- if self.config.driver != self.active_config.driver or \- self.config.driver_config != self.active_config.driver_config:+ if self._impl_pending or self.config.mode != self.active_config.mode or \+ self.config.driver != self.active_config.driver or \+ self.config.driver_config != self.active_config.driver_config:
self.impl = self.config.get_impl()
+ self._impl_pending = False
self.active_config = self.config
@synchronized
@@ -126,7 +130,16 @@ def activate(self) -> bool:
:returns False if a safe watchdog could not be configured, but is required.
"""
self.active = True
- return self._activate()+ # A configuration change might have been deferred to a keepalive that was never sent+ if not self.impl.is_running and (self._impl_pending or self.config.mode != self.active_config.mode):+ self.impl = self.config.get_impl()+ self._impl_pending = False+ if self._activate():+ return True+ # The HA loop gives up the leader key, so keepalive() won't apply configuration changes anymore. Unless a+ # running watchdog has to be disabled on demotion, reload_config() must apply them, or the node stays unhealthy.+ self.active = self.impl.is_running+ return False
def _activate(self) -> bool:
self.active_config = self.config
@@ -135,6 +148,8 @@ def _activate(self) -> bool:
logger.warning('Watchdog not supported because leader TTL %s is less than 2x loop_wait %s',
self.config.ttl, self.config.loop_wait)
self.impl = NullWatchdog()
+ # Reacquire the configured device once a configuration change fixes the timing.+ self._impl_pending = True
try:
self.impl.open()
@@ -211,8 +226,9 @@ def keepalive(self) -> None:
self.impl.keepalive()
# In case there are any pending configuration changes apply them now.
if self.active and self.config != self.active_config:
- if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF:+ if not self.impl.is_running and (self._impl_pending or self.config.mode != self.active_config.mode):
self.impl = self.config.get_impl()
+ self._impl_pending = False
self._activate()
if self.config.driver != self.active_config.driver \
or self.config.driver_config != self.active_config.driver_config:
diff --git a/tests/test_watchdog.py b/tests/test_watchdog.pyindex 62dc6ccbf3..0eb2c717ad 100644--- a/tests/test_watchdog.py+++ b/tests/test_watchdog.py@@ -164,9 +164,12 @@ def test_config_reload(self):
self.assertFalse(watchdog.is_healthy)
self.assertFalse(watchdog.activate())
+ # the HA loop gives up the leader key, so the next activation picks up the reloaded configuration
watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
self.assertFalse(watchdog.is_running)
watchdog.keepalive()
+ self.assertFalse(watchdog.is_running)+ self.assertTrue(watchdog.activate())
self.assertTrue(watchdog.is_running)
watchdog.reload_config({'ttl': 60, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
@@ -179,6 +182,62 @@ def test_config_reload(self):
self.assertTrue(watchdog.is_running)
self.assertEqual(watchdog.config.timeout, 60 // 2)
+ @patch('platform.system', Mock(return_value='Linux'))+ @patch.object(LinuxWatchdogDevice, 'is_healthy', PropertyMock(return_value=True))+ def test_reacquire_after_unsafe_timing(self):+ config = {'ttl': 20, 'loop_wait': 10, 'watchdog': {'safety_margin': 12}}+ for mode in ('automatic', 'required'):+ with self.subTest(mode=mode):+ config['watchdog']['mode'] = mode+ watchdog = Watchdog(config)+ self.assertEqual(watchdog.activate(), mode == 'automatic')+ self.assertFalse(watchdog.is_running)+ self.assertEqual(watchdog.is_healthy, mode == 'automatic')+ # automatic mode keeps the leader key, required mode gives it up+ watchdog.reload_config(dict(config, ttl=30))+ if mode == 'automatic':+ watchdog.keepalive()+ else:+ self.assertTrue(watchdog.is_healthy)+ self.assertTrue(watchdog.activate())+ self.assertTrue(watchdog.is_running)+ watchdog.disable()++ @patch('platform.system', Mock(return_value='Linux'))+ def test_mode_off_after_unsafe_timing(self):+ config = {'ttl': 20, 'loop_wait': 10, 'watchdog': {'mode': 'automatic', 'safety_margin': 12}}+ for leader in (True, False):+ with self.subTest(leader=leader):+ watchdog = Watchdog(config)+ self.assertTrue(watchdog.activate())+ if not leader:+ watchdog.disable()+ # the device must not be acquired anymore once the watchdog is turned off+ watchdog.reload_config({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'off'}})+ self.assertTrue(watchdog.activate())+ self.assertFalse(watchdog.is_running)++ @patch('platform.system', Mock(return_value='Linux'))+ @patch.object(LinuxWatchdogDevice, 'is_healthy', PropertyMock(return_value=True))+ def test_reacquire_after_mode_change(self):+ watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'off'}})+ self.assertTrue(watchdog.activate())+ watchdog.disable()+ watchdog.reload_config({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}})+ self.assertTrue(watchdog.is_healthy)+ self.assertTrue(watchdog.activate())+ self.assertTrue(watchdog.is_running)+ watchdog.disable()++ # the change was deferred to a keepalive, but the watchdog was disabled first+ watchdog.reload_config({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'off'}})+ self.assertTrue(watchdog.activate())+ watchdog.reload_config({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}})+ watchdog.disable()+ self.assertTrue(watchdog.activate())+ self.assertTrue(watchdog.is_running)+ watchdog.disable()+
class TestNullWatchdog(unittest.TestCase):