From: sgdanawe (@sgdanawe) <noreply+sgdanawe@github.com>
To: patroni/patroni <noreply+patroni-patroni@github.com>
Subject: [patroni/patroni] PR #3803: Ignore peers on an older timeline when choosing the healthiest node
Date: Wed, 18 Nov 2026 06:41:10 +0000
Message-ID: <gh-patroni-patroni-3803@github.com> (raw)
When check_timeline is enabled, a member whose timeline is behind the cluster timeline disqualifies itself and never promotes. The peer loop in _is_healthiest_node() nevertheless compared only the WAL position: a member that was ahead on LSN, because of the WAL it had written on an abandoned timeline, made every other member give up its own candidacy. If the members that could be promoted were all deferring to such a member, nobody was promoted and the cluster stayed without a leader.
Skip such a peer in the comparison, the same way is_failover_possible() already does. A peer on the cluster timeline that is ahead still wins, a peer with an unknown timeline is treated as before, and nothing changes when check_timeline is disabled.
Fixes #3802
inline patch 1: patroni/ha.py, tests/test_ha.py (+33/-1 lines) download inline-1.patchdiff --git a/patroni/ha.py b/patroni/ha.pyindex a853b7ce1..86fc0944b 100644--- a/patroni/ha.py+++ b/patroni/ha.py@@ -1401,7 +1401,8 @@ def _is_healthiest_node(self, members: Collection[Member],
logger.info('My wal position exceeds maximum replication lag')
return False # Too far behind last reported wal position on primary
- if not self.is_standby_cluster() and self.check_timeline():+ check_timeline = not self.is_standby_cluster() and self.check_timeline()+ if check_timeline:
if my_timeline is None:
logger.info('Can not figure out my timeline')
return False
@@ -1444,6 +1445,13 @@ def _is_healthiest_node(self, members: Collection[Member],
logger.warning('Primary (%s) is still alive', st.member.name)
return False
if my_wal_position < st.wal_position:
+ # With check_timeline enabled a node with the timeline behind the cluster timeline disqualifies+ # itself, therefore it will never promote and it must not make us give up our own candidacy.+ # Its WAL position is higher only because of the WAL it generated on the abandoned timeline.+ if check_timeline and st.timeline and st.timeline < cluster_timeline:+ logger.info('Ignoring %s being ahead of us: its timeline %s is behind last known '+ 'cluster timeline %s', st.member.name, st.timeline, cluster_timeline)+ continue
nodes_ahead += 1
logger.info('Wal position of %s is ahead of my wal position', st.member.name)
# In synchronous mode the former leader might be still accessible and even be ahead of us.
diff --git a/tests/test_ha.py b/tests/test_ha.pyindex 47ded458d..93725c94c 100644--- a/tests/test_ha.py+++ b/tests/test_ha.py@@ -1308,6 +1308,30 @@ def test__is_healthiest_node(self):
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', None, None, 'dc1'))
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+ def test__is_healthiest_node_peer_on_older_timeline(self):+ self.p.is_primary = false+ self.ha.cluster = get_cluster_initialized_without_leader()+ global_config.update(self.ha.cluster)+ members = self.ha.old_cluster.members+ # the peer is ahead on LSN, but it is on a timeline behind the cluster timeline+ with patch.object(Cluster, 'timeline', PropertyMock(return_value=3)), \+ patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=3):+ # the peer can't be promoted (check_timeline), so it must not make us defer+ self.ha.fetch_node_status = get_node_status(wal_position=11, timeline=2)+ with patch('patroni.ha.logger.info') as mock_info:+ self.assertTrue(self.ha._is_healthiest_node(members))+ self.assertIn('Ignoring %s being ahead of us', mock_info.call_args_list[0][0][0])+ # the peer is on the cluster timeline and ahead: defer to it+ self.ha.fetch_node_status = get_node_status(wal_position=11, timeline=3)+ self.assertFalse(self.ha._is_healthiest_node(members))+ # unknown timeline of the peer: keep the old behaviour+ self.ha.fetch_node_status = get_node_status(wal_position=11, timeline=0)+ self.assertFalse(self.ha._is_healthiest_node(members))+ # check_timeline is disabled: timelines are not taken into account+ self.ha.fetch_node_status = get_node_status(wal_position=11, timeline=2)+ with patch.object(Ha, 'check_timeline', Mock(return_value=False)):+ self.assertFalse(self.ha._is_healthiest_node(members))+
def test_fetch_node_status(self):
member = Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni';})
self.ha.fetch_node_status(member)