From: egor-voynov-aiven (@egor-voynov-aiven) <noreply+egor-voynov-aiven@github.com>
To: patroni/patroni <noreply+patroni-patroni@github.com>
Subject: [patroni/patroni] PR #3779: Disable the watchdog after a failed pre_promote
Date: Fri, 13 Nov 2026 14:45:05 +0000
Message-ID: <gh-patroni-patroni-3779@github.com> (raw)
When a replica wins the leader race, `enforce_primary_role()` activates the watchdog before the promotion starts, and `pre_promote` runs as the first step of the promotion. If the script exits non-zero, the node releases the leader key and skips the leader race for ttl seconds, but the watchdog stays armed. Nothing feeds it anymore, because keepalive() is only called after the leader key is renewed. That is earlier than the end of the backoff, so the retry the code prepares for can never happen.
This PR disables the watchdog in that branch when Postgres is not running as primary. The node stays up as a replica, honours the backoff, and activates the watchdog again if it wins the race later. A node that is already running as primary keeps its watchdog as before.
inline patch 1: patroni/ha.py, tests/test_ha.py (+21/-3 lines) download inline-1.patchdiff --git a/patroni/ha.py b/patroni/ha.pyindex a853b7ce1..b56d61804 100644--- a/patroni/ha.py+++ b/patroni/ha.py@@ -1218,6 +1218,10 @@ def enforce_primary_role(self, message: str, promote_message: str) -> str:
logger.warning("Releasing the leader key voluntarily because the pre-promote script failed")
self._released_leader_key_timestamp = time.monotonic()
self.release_leader_key_voluntarily()
+ if not self.state_handler.is_primary():+ # the leader key is gone and nobody will feed the watchdog anymore,+ # disarm it to avoid a reset while the node is in the backoff as a replica+ self.watchdog.disable()
# discard the result of the failed pre-promote script to be able to re-try promote
self._async_response.reset()
return 'Promotion cancelled because the pre-promote script failed'
diff --git a/tests/test_ha.py b/tests/test_ha.pyindex 47ded458d..fce96f504 100644--- a/tests/test_ha.py+++ b/tests/test_ha.py@@ -425,9 +425,23 @@ def test_promotion_cancelled_after_pre_promote_failed(self):
self.p.is_primary = false
self.p._pre_promote = false
self.ha._is_healthiest_node = true
- self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')- self.assertEqual(self.ha.run_cycle(), 'Promotion cancelled because the pre-promote script failed')- self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')+ with patch.object(Watchdog, 'disable') as mock_disable:+ self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')+ mock_disable.assert_not_called()+ self.assertEqual(self.ha.run_cycle(), 'Promotion cancelled because the pre-promote script failed')+ # the leader key is released, nobody feeds the watchdog anymore+ mock_disable.assert_called_once()+ self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')++ def test_pre_promote_failed_keeps_watchdog_on_primary(self):+ self.p.is_primary = false+ self.p._pre_promote = false+ self.ha._is_healthiest_node = true+ with patch.object(Watchdog, 'disable') as mock_disable:+ self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')+ self.p.is_primary = true+ self.assertEqual(self.ha.run_cycle(), 'Promotion cancelled because the pre-promote script failed')+ mock_disable.assert_not_called()
def test_lost_leader_lock_during_promote(self):
with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):