pg.ddx.io  pgbouncer/pgbouncer GitHub issues and pull requests  
help / color / mirror / Atom feed
[pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
7+ messages / 2 participants
[nested] [flat]

* [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-14 21:10  "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
  0 siblings, 0 replies; 7+ messages in thread

From: cosgroveb (@cosgroveb) @ 2026-11-14 21:10 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

A concurrent `RESUME db` can leave the client running `PAUSE db` blocked forever.

Track the pending admin command and database target.

`RESUME` cancels only matching `PAUSE` requests.

Add regressions for matching and unrelated databases.

Fixes #715.

inline patch 1: doc/usage.md, include/admin.h, include/bouncer.h and 3 more (+136/-13 lines) download inline-1.patchdiff --git a/doc/usage.md b/doc/usage.md index 5722274f8ab3..cbd25158de9d 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -715,6 +715,8 @@ mode (in transaction pooling mode, the transaction must complete, in statement m the statement must complete, and in session pooling mode the client must disconnect). The command will not return before all server connections have been disconnected. To be used at the time of database restart. +A matching **RESUME** issued before the pause completes cancels the pending +**PAUSE**, which then returns an error. If database name is given, only that database will be paused. diff --git a/include/admin.h b/include/admin.h index b6747f430aba..e39352e291f5 100644 --- a/include/admin.h +++ b/include/admin.h @@ -21,6 +21,7 @@ bool admin_post_login(PgSocket *client) _MUSTCHECK; void admin_setup(void); bool admin_error(PgSocket *console, const char *fmt, ...) _PRINTF(2, 3) /* _MUSTCHECK */; void admin_pause_done(void); +void admin_pause_cancel(const char *target); void admin_wait_close_done(void); bool admin_flush(PgSocket *admin, PktBuf *buf, const char *desc) /* _MUSTCHECK */; bool admin_ready(PgSocket *admin, const char *desc) _MUSTCHECK; diff --git a/include/bouncer.h b/include/bouncer.h index 88ea46eb35cf..539e778451e8 100644 --- a/include/bouncer.h +++ b/include/bouncer.h @@ -93,6 +93,12 @@ enum PauseMode { P_PAUSE = 1 /* wait for client to finish work */ }; +enum AdminWaitType { + ADMIN_WAIT_NONE, + ADMIN_WAIT_PAUSE, + ADMIN_WAIT_CLOSE, +}; + enum ShutDownMode { /* just stay running */ SHUTDOWN_NONE = 0, @@ -728,7 +734,8 @@ struct PgSocket { bool admin_user : 1; /* console client: has admin rights */ bool own_user : 1; /* console client: client with same uid on unix socket */ - bool wait_for_response : 1; /* console client: waits for completion of PAUSE cmd */ + enum AdminWaitType admin_wait; /* console client: pending asynchronous command */ + char pause_target[MAX_DBNAME]; /* console client: database targeted by pending PAUSE; empty for global */ bool wait_sslchar : 1; /* server: waiting for ssl response: S/N */ /* server: received an ErrorResponse, waiting for ReadyForQuery to clear diff --git a/src/admin.c b/src/admin.c index 5cdb96f2e311..e0b7728c5f1b 100644 --- a/src/admin.c +++ b/src/admin.c @@ -998,6 +998,7 @@ static bool admin_cmd_resume(PgSocket *admin, const char *arg) return admin_error(admin, "pooler is shutting down"); } else if (cf_pause_mode != P_NONE) { cf_pause_mode = P_NONE; + admin_pause_cancel(""); } else { return admin_error(admin, "pooler is not paused"); } @@ -1009,6 +1010,7 @@ static bool admin_cmd_resume(PgSocket *admin, const char *arg) if (!db->db_paused) return admin_error(admin, "database %s is not paused", arg); db->db_paused = false; + admin_pause_cancel(db->name); } return admin_ready(admin, "RESUME"); } @@ -1025,7 +1027,8 @@ static bool admin_cmd_pause(PgSocket *admin, const char *arg) if (!arg[0]) { log_info("PAUSE command issued"); cf_pause_mode = P_PAUSE; - admin->wait_for_response = true; + admin->admin_wait = ADMIN_WAIT_PAUSE; + admin->pause_target[0] = '\0'; } else { PgDatabase *db; log_info("PAUSE '%s' command issued", arg); @@ -1035,10 +1038,12 @@ static bool admin_cmd_pause(PgSocket *admin, const char *arg) if (db == admin->pool->db) return admin_error(admin, "cannot pause admin db: %s", arg); db->db_paused = true; - if (count_db_active(db) > 0) - admin->wait_for_response = true; - else + if (count_db_active(db) > 0) { + admin->admin_wait = ADMIN_WAIT_PAUSE; + strlcpy(admin->pause_target, db->name, sizeof(admin->pause_target)); + } else { return admin_ready(admin, "PAUSE"); + } } return true; @@ -1264,7 +1269,7 @@ static bool admin_cmd_wait_close(PgSocket *admin, const char *arg) active += count_db_active(db); } if (active > 0) - admin->wait_for_response = true; + admin->admin_wait = ADMIN_WAIT_CLOSE; else return admin_ready(admin, "WAIT_CLOSE"); } else { @@ -1278,7 +1283,7 @@ static bool admin_cmd_wait_close(PgSocket *admin, const char *arg) return admin_error(admin, "cannot wait in admin db: %s", arg); db->db_wait_close = true; if (count_db_active(db) > 0) - admin->wait_for_response = true; + admin->admin_wait = ADMIN_WAIT_CLOSE; else return admin_ready(admin, "WAIT_CLOSE"); } @@ -1686,7 +1691,7 @@ void admin_pause_done(void) statlist_for_each_safe(item, &admin_pool->active_client_list, tmp) { admin = container_of(item, PgSocket, head); - if (!admin->wait_for_response) + if (admin->admin_wait != ADMIN_WAIT_PAUSE) continue; switch (cf_pause_mode) { @@ -1706,7 +1711,29 @@ void admin_pause_done(void) if (!res) disconnect_client(admin, false, "dead admin"); else - admin->wait_for_response = false; + admin->admin_wait = ADMIN_WAIT_NONE; + } +} + +/* answer pending PAUSE commands canceled by RESUME */ +void admin_pause_cancel(const char *target) +{ + struct List *item, *tmp; + PgSocket *admin; + bool res; + + statlist_for_each_safe(item, &admin_pool->active_client_list, tmp) { + admin = container_of(item, PgSocket, head); + if (admin->admin_wait != ADMIN_WAIT_PAUSE) + continue; + if (strcmp(admin->pause_target, target) != 0) + continue; + + res = admin_error(admin, "PAUSE command canceled by RESUME"); + if (!res) + disconnect_client(admin, false, "dead admin"); + else + admin->admin_wait = ADMIN_WAIT_NONE; } } @@ -1718,7 +1745,7 @@ void admin_wait_close_done(void) statlist_for_each_safe(item, &admin_pool->active_client_list, tmp) { admin = container_of(item, PgSocket, head); - if (!admin->wait_for_response) + if (admin->admin_wait != ADMIN_WAIT_CLOSE) continue; res = admin_ready(admin, "WAIT_CLOSE"); @@ -1726,7 +1753,7 @@ void admin_wait_close_done(void) if (!res) disconnect_client(admin, false, "dead admin"); else - admin->wait_for_response = false; + admin->admin_wait = ADMIN_WAIT_NONE; } } @@ -1734,7 +1761,7 @@ void admin_wait_close_done(void) void admin_handle_cancel(PgSocket *admin) { /* weird, but no reason to fail */ - if (!admin->wait_for_response) + if (admin->admin_wait == ADMIN_WAIT_NONE) slog_warning(admin, "admin cancel request for non-waiting client?"); if (cf_shutdown) diff --git a/src/main.c b/src/main.c index f41130aeaf95..258f557f4197 100644 --- a/src/main.c +++ b/src/main.c @@ -593,6 +593,7 @@ static void handle_sigusr2(int sock, short flags, void *arg) case P_PAUSE: log_info("got SIGUSR2, continuing from PAUSE"); cf_pause_mode = P_NONE; + admin_pause_cancel(""); break; case P_NONE: log_info("got SIGUSR2, but not paused"); diff --git a/test/test_operations.py b/test/test_operations.py index 8e93282a4ca4..e2feec39486e 100644 --- a/test/test_operations.py +++ b/test/test_operations.py @@ -3,8 +3,9 @@ import psycopg import pytest +from psycopg.rows import dict_row -from .utils import WINDOWS +from .utils import WINDOWS, wait_until async def test_pause_resume(bouncer): @@ -19,6 +20,90 @@ async def test_pause_resume(bouncer): await task +async def wait_for_admin_row(bouncer, query, **expected): + for _ in wait_until(f"{query} has no row matching {expected}", interval=0): + rows = await bouncer.aadmin(query, row_factory=dict_row) + if any(expected.items() <= row.items() for row in rows): + return + await asyncio.sleep(0.1) + + +async def test_resume_cancels_pending_pause(bouncer): + """RESUME p0 must answer a pending PAUSE p0, but not WAIT_CLOSE p0 (#715).""" + sleep_task = bouncer.asleep(5, dbname="p0") + await wait_for_admin_row(bouncer, "SHOW SERVERS", database="p0", state="active") + await bouncer.aadmin("RECONNECT p0") + wait_close_task = bouncer.aadmin("WAIT_CLOSE p0") + for _ in wait_until("WAIT_CLOSE p0 was not issued", interval=0): + if "WAIT_CLOSE 'p0' command issued" in bouncer.log_path.read_text(): + break + await asyncio.sleep(0.1) + pause_task = bouncer.aadmin("PAUSE p0") + await wait_for_admin_row(bouncer, "SHOW DATABASES", name="p0", paused=1) + + await bouncer.aadmin("RESUME p0") + + with pytest.raises( + psycopg.errors.ProtocolViolation, match=r"^PAUSE command canceled by RESUME$" + ): + await asyncio.wait_for(pause_task, timeout=5) + _, pending = await asyncio.wait([wait_close_task], timeout=0.5) + assert wait_close_task in pending, "RESUME p0 completed WAIT_CLOSE p0" + await sleep_task + await wait_close_task + + +async def test_resume_does_not_complete_foreign_pending_pause(bouncer): + sleep_task = bouncer.asleep(5, dbname="p1") + await wait_for_admin_row(bouncer, "SHOW SERVERS", database="p1", state="active") + pause_task = bouncer.aadmin("PAUSE p1") + await wait_for_admin_row(bouncer, "SHOW DATABASES", name="p1", paused=1) + + await bouncer.aadmin("PAUSE p0") + await bouncer.aadmin("RESUME p0") + + _, pending = await asyncio.wait([pause_task], timeout=0.5) + assert pause_task in pending, "RESUME p0 completed or canceled PAUSE p1" + await sleep_task + await pause_task + + +async def test_global_resume_cancels_pending_global_pause(bouncer): + sleep_task = bouncer.asleep(5, dbname="p1") + await wait_for_admin_row(bouncer, "SHOW SERVERS", database="p1", state="active") + db_pause_task = bouncer.aadmin("PAUSE p1") + await wait_for_admin_row(bouncer, "SHOW DATABASES", name="p1", paused=1) + pause_task = bouncer.aadmin("PAUSE") + await wait_for_admin_row(bouncer, "SHOW STATE", key="paused", value="yes") + + await bouncer.aadmin("RESUME") + + with pytest.raises( + psycopg.errors.ProtocolViolation, match=r"^PAUSE command canceled by RESUME$" + ): + await asyncio.wait_for(pause_task, timeout=5) + _, pending = await asyncio.wait([db_pause_task], timeout=0.5) + assert db_pause_task in pending, "global RESUME canceled PAUSE p1" + await sleep_task + await db_pause_task + + +@pytest.mark.skipif("WINDOWS", reason="Windows does not have SIGUSR2") +async def test_sigusr2_cancels_pending_pause(bouncer): + sleep_task = bouncer.asleep(5, dbname="p0") + await wait_for_admin_row(bouncer, "SHOW SERVERS", database="p0", state="active") + pause_task = bouncer.aadmin("PAUSE") + await wait_for_admin_row(bouncer, "SHOW STATE", key="paused", value="yes") + + bouncer.sigusr2() + + with pytest.raises( + psycopg.errors.ProtocolViolation, match=r"^PAUSE command canceled by RESUME$" + ): + await asyncio.wait_for(pause_task, timeout=5) + await sleep_task + + def test_enable_disable(bouncer): bouncer.test() bouncer.admin("disable p0")
^ permalink raw [nested|flat] 7+ messages in thread

* Re: [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-14 21:56  "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
  5 siblings, 0 replies; 7+ messages in thread

From: cosgroveb (@cosgroveb) @ 2026-11-14 21:56 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

I can't believe I forgot about this. We used to coordinate more than 100 bouncers in production while promoting a synchronous standby making this a very real edge case we've hit a few times.

This is attenuated for me by internal architecture improvements but still real.

^ permalink  raw  [nested|flat] 7+ messages in thread

* Re: [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-15 02:43  "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
  5 siblings, 0 replies; 7+ messages in thread

From: cosgroveb (@cosgroveb) @ 2026-11-15 02:43 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

@cosgroveb renamed this.

^ permalink  raw  [nested|flat] 7+ messages in thread

* Re: [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-15 02:58  "ritojoao (@ritojoao)" <noreply+ritojoao@github.com>
  5 siblings, 0 replies; 7+ messages in thread

From: ritojoao (@ritojoao) @ 2026-11-15 02:58 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

Hi, and thanks for picking this up. I tested this PR (ab59242) against current master (ac3bb6d), building both from source in a postgres:17 container.

- The three-session sequence from the issue (a running `pg_sleep`, `PAUSE db`, then `RESUME db`): on master the PAUSE client never gets an answer. With the PR it returns `PAUSE command canceled by RESUME` right away. The same holds for global `PAUSE` and `RESUME`.
- With two databases paused and both PAUSE commands waiting, `RESUME` on one of them cancels only that PAUSE. The other one keeps waiting and completes once its database is idle, and that database stays paused.
- A concurrent `WAIT_CLOSE` is not touched by `RESUME`. On master, a waiting `PAUSE` could be answered with the `WAIT_CLOSE` completion, which the PR avoids.
- Of the 4 new tests, 3 fail on master and 1 passes on both. `test_operations.py` and `test_admin.py` pass with the PR (28 passed).

I also pressed ^C on a waiting PAUSE (db-level and global). The behavior is the same as master, so this PR does not change it.

What I did not test: the full pytest suite, transaction/statement pool modes, TLS/auth, non-Linux platforms, several waiters on the same database, and SIGUSR2 outside the PR's own tests. Thanks again for the fix.


^ permalink  raw  [nested|flat] 7+ messages in thread

* Re: [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-15 17:06  "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
  5 siblings, 0 replies; 7+ messages in thread

From: cosgroveb (@cosgroveb) @ 2026-11-15 17:06 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

Hey @ritojoao,

We don't document any particular behavior on ^C. psql does the right thing and sends a protocol `CancelRequest`, which is handled in part by pgbouncer even though it never replies to the admin console. 

It looks like this was meant to work the way it [obviously should](https://github.com/pgbouncer/pgbouncer/commit/8613ca9320d1deacf30ad5b24d93a712c2ef41ee#diff-378f5127...). That changed with #147 maybe?

It looks like the reviewer and author there got close but didn't quite consider this case? https://github.com/pgbouncer/pgbouncer/pull/147#issuecomment-254341812

Nevertheless, the root cause here is a separate bug IMHO.

^ permalink  raw  [nested|flat] 7+ messages in thread

* Re: [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-15 17:10  "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
  5 siblings, 0 replies; 7+ messages in thread

From: cosgroveb (@cosgroveb) @ 2026-11-15 17:10 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

The mix of:

* admin connection
* protocol `CancelRequest`
* and thus no text "RESUME" command

Is what makes your ^C use-case problematic currently. 

^ permalink  raw  [nested|flat] 7+ messages in thread

* Re: [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME
@ 2026-11-16 00:42  "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
  5 siblings, 0 replies; 7+ messages in thread

From: cosgroveb (@cosgroveb) @ 2026-11-16 00:42 UTC (permalink / raw)
  To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>

@ritojoao I opened a new issue for the ^C bug since while very closely related is different. I want to make sure we're on the right track with the fix in this PR first.

^ permalink  raw  [nested|flat] 7+ messages in thread


end of thread, other threads:[~2026-11-16 00:42 UTC | newest]

Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2026-11-14 21:10 [pgbouncer/pgbouncer] PR #1708: Answer pending PAUSE commands canceled by RESUME "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
2026-11-14 21:56 ` "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
2026-11-15 02:43 ` "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
2026-11-15 02:58 ` "ritojoao (@ritojoao)" <noreply+ritojoao@github.com>
2026-11-15 17:06 ` "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
2026-11-15 17:10 ` "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>
2026-11-16 00:42 ` "cosgroveb (@cosgroveb)" <noreply+cosgroveb@github.com>

This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox