From: ritojoao (@ritojoao) <noreply+ritojoao@github.com>
To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>
Subject: [pgbouncer/pgbouncer] PR #1700: Fix queued peer cancel requests not being retried
Date: Tue, 10 Nov 2026 20:32:33 +0000
Message-ID: <gh-pgbouncer-pgbouncer-1700@github.com> (raw)
Fixes #1642
Hello, and thank you for maintaining PgBouncer.
**What was wrong**
When many cancel requests had to be forwarded to the same peer at the same time, only one connection attempt per peer pool can be in progress. The other requests stayed in `waiting_cancel_req_list`. `per_loop_maint()` retries queued cancel requests for regular pools, but it did not do this for `peer_pool_list`, so nothing started a new attempt after the first one finished. The requests only ended when `cancel_wait_timeout` expired, while the queries kept running.
**What changed**
`per_loop_maint()` now also calls `launch_new_connection()` for every peer pool that has queued cancel requests. This follows the idea suggested in the issue.
I added `test_concurrent_peer_cancel` to `test/test_peering.py`. It starts two peers with their own ports, opens 12 raw protocol client connections on each, runs `select pg_sleep(20)` on all of them, and then sends every client's CancelRequest to the other peer, all at the same moment. It checks that every query ends with SQLSTATE 57014 within 5 seconds. Without the change the requests wait for `cancel_wait_timeout` (10 seconds).
**How tested**
Built on Linux (postgres:17 container, libevent 2.1, OpenSSL 3, PostgreSQL 17) with `./autogen.sh && ./configure && make`, tests run with `pytest -p no:cacheprovider`.
- `pytest test/test_peering.py -k concurrent_peer_cancel`, 3 runs without the `src/janitor.c` change: fails every time, the cancel requests only finish after about 10.2 seconds (`cancel_wait_timeout`).
- Same command, 3 runs with the change: passes every time (the test body takes about 1.5 seconds).
- `pytest test/test_peering.py test/test_cancel.py` with the change: 9 passed.
- `ruff format --check` and `ruff check` on `test/test_peering.py`: clean. I did not run the C formatter (uncrustify).
inline patch 1: src/janitor.c, test/test_peering.py (+139/-0 lines) download inline-1.patchdiff --git a/src/janitor.c b/src/janitor.cindex fefa689c1b3d..fa305a1bfad9 100644--- a/src/janitor.c+++ b/src/janitor.c@@ -285,6 +285,17 @@ void per_loop_maint(void)
}
}
+ /*+ * Only one connection attempt per pool is in progress at a time, so+ * queued cancel requests for peers need to be retried once the earlier+ * attempt has finished, just like it is done for regular pools above.+ */+ statlist_for_each(item, &peer_pool_list) {+ pool = container_of(item, PgPool, head);+ if (!statlist_empty(&pool->waiting_cancel_req_list))+ launch_new_connection(pool, /* evict_if_needed= */ true);+ }+
switch (cf_pause_mode) {
case P_PAUSE:
if (!active_count)
diff --git a/test/test_peering.py b/test/test_peering.pyindex dcf172f6f76b..562b948c1555 100644--- a/test/test_peering.py+++ b/test/test_peering.py@@ -1,4 +1,7 @@
import asyncio
+import socket+import struct+import threading
import time
from concurrent.futures import ThreadPoolExecutor
@@ -136,3 +139,128 @@ async def test_rolling_restart_sigterm(peers):
# Now that the connection is closed, peer[1] should exit automatically.
await peers[1].wait_for_exit()
assert not peers[1].running()
+++@pytest.fixture+async def separate_peers(pg, tmp_path):+ # Two peers that each have their own port, so that a cancel request can be+ # sent to a specific peer. Peers are reached over TCP, like they would be+ # when they run on different hosts.+ peers: dict[int, Bouncer] = {+ 1: Bouncer(pg, tmp_path / "bouncer1"),+ 2: Bouncer(pg, tmp_path / "bouncer2"),+ }++ for own_index, bouncer in peers.items():+ with bouncer.ini_path.open("a") as f:+ f.write(f"peer_id={own_index}\n")+ f.write("max_client_conn=200\n")+ f.write("default_pool_size=40\n")+ f.write("[peers]\n")+ for other_index, peer in peers.items():+ if own_index == other_index:+ continue+ f.write(f"{other_index} = host=127.0.0.1 port={peer.port}\n")++ await asyncio.gather(*[p.start() for p in peers.values()])++ yield peers++ await asyncio.gather(*[p.cleanup() for p in peers.values()])+++def read_exactly(sock, n):+ data = b""+ while len(data) < n:+ chunk = sock.recv(n - len(data))+ if not chunk:+ raise EOFError("connection closed")+ data += chunk+ return data+++def read_message(sock):+ msg_type = read_exactly(sock, 1)+ (length,) = struct.unpack("!I", read_exactly(sock, 4))+ return msg_type, read_exactly(sock, length - 4)+++def raw_connect(bouncer):+ """Connect using the raw protocol and return (socket, cancel key)"""+ sock = socket.create_connection(("127.0.0.1", bouncer.port), timeout=30)+ params = b"user\x00bouncer\x00database\x00p3\x00\x00"+ sock.sendall(struct.pack("!II", 8 + len(params), 196608) + params)+ key = None+ while True:+ msg_type, payload = read_message(sock)+ if msg_type == b"K":+ key = payload+ elif msg_type == b"Z":+ break+ elif msg_type == b"E":+ raise RuntimeError(f"login failed: {payload!r}")+ assert key is not None+ return sock, key+++def send_query(sock, query):+ payload = query.encode() + b"\x00"+ sock.sendall(b"Q" + struct.pack("!I", 4 + len(payload)) + payload)+++def send_cancel(bouncer, key):+ with socket.create_connection(("127.0.0.1", bouncer.port), timeout=30) as sock:+ sock.sendall(struct.pack("!II", 8 + len(key), 80877102) + key)+ # PgBouncer closes the connection once the request was handled (or+ # when it gave up on it)+ assert sock.recv(1) == b""+++def query_error_code(sock):+ """Read until the query is done, return the SQLSTATE of its error (if any)"""+ code = None+ while True:+ msg_type, payload = read_message(sock)+ if msg_type == b"E":+ fields = payload.split(b"\x00")+ code = next(f[1:] for f in fields if f.startswith(b"C")).decode()+ elif msg_type == b"Z":+ return code+++def test_concurrent_peer_cancel(separate_peers):+ # Cancel requests that were sent to the wrong peer have to be forwarded to+ # the right one. Only one connection attempt per peer pool is in progress+ # at a time, so when many requests arrive at once, the others should be+ # picked up afterwards instead of waiting until cancel_wait_timeout (10+ # seconds by default).+ per_peer = 12+ clients = []+ try:+ for owner in (1, 2):+ for _ in range(per_peer):+ sock, key = raw_connect(separate_peers[owner])+ clients.append((owner, sock, key))+ for _, sock, _ in clients:+ send_query(sock, "select pg_sleep(20)")+ time.sleep(1)++ # Send every cancel request to the peer that does not own the query,+ # all at the same moment.+ barrier = threading.Barrier(len(clients))++ def cancel(owner, key):+ barrier.wait()+ send_cancel(separate_peers[3 - owner], key)++ start = time.monotonic()+ with ThreadPoolExecutor(max_workers=len(clients)) as pool:+ cancels = [pool.submit(cancel, owner, key) for owner, _, key in clients]+ for c in cancels:+ c.result()+ assert time.monotonic() - start < 5+ for _, sock, _ in clients:+ assert query_error_code(sock) == "57014"+ finally:+ for _, sock, _ in clients:+ sock.close()