pg.ddx.io  pgbouncer/pgbouncer GitHub issues and pull requests  
help / color / mirror / Atom feed
From: eulerto (@eulerto) <noreply+eulerto@github.com>
To: pgbouncer/pgbouncer <noreply+pgbouncer-pgbouncer@github.com>
Subject: [pgbouncer/pgbouncer] PR #1698: Fix use-after-free of server TLS config on SIGHUP
Date: Thu, 05 Nov 2026 00:56:19 +0000
Message-ID: <gh-pgbouncer-pgbouncer-1698@github.com> (raw)

A SIGHUP reload runs `sbuf_tls_setup()`, which unconditionally frees the
old `server_connect_conf` (`src/sbuf.c`) and builds a new one. However
`tls_configure()` only stored a borrowed pointer (`ctx->config`), so a
PgBouncer->server TLS handshake that is parked inside `SSL_connect()`
(`SBUF_TLS_DO_HANDSHAKE`) keeps pointing at the freed config. When that
handshake later completes, `tls_handshake_client()` reads
`ctx->config->verify_name` from freed memory. With server_tls_sslmode=
verify-ca the `servername` is NULL, so if the freed verify_name reads
non-zero PgBouncer calls `tls_check_name(ctx, cert, NULL)`, which reaches
`inet_pton(AF_INET, NULL)` -> `strlen(NULL)` -> Kaboom. This is sporadic
because it only crashes when the freed block is reused before the
handshake resumes.

Fix it by reference counting `struct tls_config`, matching upstream
libtls: `tls_config_new()` starts at `refcount` 1, `tls_configure()` takes a
reference, and `usual_tls_free()` releases it, so an in-flight handshake
keeps the old config alive until the connection is gone even though the
global pointer has already been swapped.

There is a test case that I included in a separate commit. I didn't decide if
it is worth including it.

Fixes #1697

inline patch 1: lib/usual/tls/tls.c, lib/usual/tls/tls_config.c, lib/usual/tls/tls_internal.h and 2 more (+238/-0 lines) download inline-1.patchdiff --git a/lib/usual/tls/tls.c b/lib/usual/tls/tls.c index 6e806a6457be..38e7aea9a2fc 100644 --- a/lib/usual/tls/tls.c +++ b/lib/usual/tls/tls.c @@ -229,6 +229,8 @@ struct tls *tls_new(void) return (NULL); ctx->config = tls_config_default; + if (ctx->config != NULL) + ctx->config->refcount++; tls_reset(ctx); @@ -240,6 +242,13 @@ int tls_configure(struct tls *ctx, struct tls_config *config) if (config == NULL) config = tls_config_default; + /* + * Take a reference on the new config before releasing the old one (they + * may be the same object), and keep the reference for as long as this + * context points at the config. usual_tls_free() releases it. + */ + config->refcount++; + tls_config_free(ctx->config); ctx->config = config; if ((ctx->flags & TLS_SERVER) != 0) @@ -531,6 +540,8 @@ void usual_tls_free(struct tls *ctx) if (ctx == NULL) return; tls_reset(ctx); + tls_config_free(ctx->config); + ctx->config = NULL; free(ctx); } diff --git a/lib/usual/tls/tls_config.c b/lib/usual/tls/tls_config.c index 9c3232331858..7a62dfedf438 100644 --- a/lib/usual/tls/tls_config.c +++ b/lib/usual/tls/tls_config.c @@ -114,6 +114,8 @@ struct tls_config *tls_config_new(void) if ((config = calloc(1, sizeof(*config))) == NULL) return (NULL); + config->refcount = 1; + if ((config->keypair = tls_keypair_new()) == NULL) goto err; @@ -150,6 +152,15 @@ void tls_config_free(struct tls_config *config) if (config == NULL) return; + /* + * This decrements the reference count, and frees the config if the count + * has thereby gone to zero. + */ + Assert(config->refcount > 0); + config->refcount--; + if (config->refcount > 0) + return; + for (kp = config->keypair; kp != NULL; kp = nkp) { nkp = kp->next; tls_keypair_free(kp); diff --git a/lib/usual/tls/tls_internal.h b/lib/usual/tls/tls_internal.h index 0ecbd9db0fdd..4263a8200fab 100644 --- a/lib/usual/tls/tls_internal.h +++ b/lib/usual/tls/tls_internal.h @@ -49,6 +49,13 @@ struct tls_keypair { struct tls_config { struct tls_error error; + /* + * A tls_config is a shared resource, referenced by every tls context + * configured with it, so it is freed only once the last user releases its + * reference. + */ + int refcount; + const char *ca_file; const char *ca_path; char *ca_mem; diff --git a/lib/usual/tls/tls_verify.c b/lib/usual/tls/tls_verify.c index bfb8e142e531..1cdb3c85d643 100644 --- a/lib/usual/tls/tls_verify.c +++ b/lib/usual/tls/tls_verify.c @@ -245,6 +245,8 @@ int tls_check_name(struct tls *ctx, X509 *cert, const char *name) { int rv; + Assert(name != NULL); + rv = tls_check_subject_altname(ctx, cert, name); if (rv == 0 || rv == -2) return rv; diff --git a/test/test_ssl_sighup_reload.py b/test/test_ssl_sighup_reload.py new file mode 100644 index 000000000000..28c8f95474bf --- /dev/null +++ b/test/test_ssl_sighup_reload.py @@ -0,0 +1,207 @@ +import asyncio +import socket +import ssl +import struct +import threading +import time + +import pytest + +from .utils import TLS_SUPPORT, WINDOWS, Bouncer + +if not TLS_SUPPORT: + pytest.skip(allow_module_level=True) + +if WINDOWS: + # SIGHUP does not exist on Windows. + pytest.skip(allow_module_level=True) + + +SSL_REQUEST_CODE = 80877103 + + +class StallingTLSBackend: + """A fake PostgreSQL backend that accepts pgBouncer's SSLRequest, replies + 'S', and then *stalls* before completing the TLS handshake. + + The stall leaves pgBouncer parked inside SSL_connect() while it still holds + a pointer to the server TLS config (server_connect_conf). That is the window + in which we SIGHUP pgBouncer so the reload frees that config underneath the + in-flight handshake (issue #1697). + """ + + def __init__(self, certfile, keyfile): + self.certfile = str(certfile) + self.keyfile = str(keyfile) + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(5) + self.port = self.sock.getsockname()[1] + # Set once the backend has sent 'S' on the first connection and is + # about to stall -> the handshake is now in flight in pgBouncer. + self.stalling = threading.Event() + # The test sets this after it has sent SIGHUP, telling the backend to + # go ahead and finish the handshake. + self.proceed = threading.Event() + self._stop = False + self._thread = threading.Thread(target=self._serve, daemon=True) + + def start(self): + self._thread.start() + + def stop(self): + self._stop = True + self.proceed.set() + try: + self.sock.close() + except OSError: + pass + + def _serve(self): + first = True + while not self._stop: + try: + conn, _ = self.sock.accept() + except OSError: + return + try: + self._handle(conn, first) + except (OSError, ssl.SSLError): + pass + first = False + + def _handle(self, conn, first): + # Note: ssl.wrap_socket() detaches the original socket and returns a new + # SSLSocket owning the fd, so we track the currently-live socket in + # `sock` and close exactly that one in the finally. + sock = conn + try: + hdr = _recv_exactly(sock, 8) + if hdr is None: + return + _, code = struct.unpack("!II", hdr) + if code != SSL_REQUEST_CODE: + return + sock.sendall(b"S") + if first: + # Announce the stall window and wait until the test reloaded. + self.stalling.set() + self.proceed.wait(timeout=10) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(self.certfile, self.keyfile) + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + sock = ctx.wrap_socket(sock, server_side=True) + # Hold the connection open briefly so pgBouncer finishes its side of + # the handshake (the point at which it reads the TLS config). + time.sleep(1) + try: + sock.unwrap() + except OSError: + pass + finally: + try: + sock.close() + except OSError: + pass + + +def _recv_exactly(conn, n): + buf = b"" + while len(buf) < n: + chunk = conn.recv(n - len(buf)) + if not chunk: + return None + buf += chunk + return buf + + +def _startup_packet(user, database): + body = b"\x00\x03\x00\x00" # protocol version 3.0 + for key, value in (("user", user), ("database", database)): + body += key.encode() + b"\x00" + value.encode() + b"\x00" + body += b"\x00" + return struct.pack("!I", len(body) + 4) + body + + +def _trigger_backend_connection(host, port): + """Open a client connection to pgBouncer and send a startup packet. With + auth_type=trust this makes pgBouncer open a backend server connection and + start the TLS handshake to our stalling fake backend.""" + cli = socket.create_connection((host, port)) + cli.sendall(_startup_packet("postgres", "stalldb")) + return cli + + +async def test_sighup_during_server_tls_handshake(pg, tmp_path, cert_dir): + """Regression test for issue #1697. + + A SIGHUP (reload) that lands while pgBouncer is in the middle of a + server-side TLS handshake must not crash. Before the fix, the reload freed + server_connect_conf while the in-flight handshake still pointed at it, and + pgBouncer dereferenced the freed config (ctx->config->verify_name) when the + handshake completed -> use-after-free / segfault. + """ + ca = cert_dir / "TestCA1" / "ca.crt" + cert = cert_dir / "TestCA1" / "sites" / "01-localhost.crt" + key = cert_dir / "TestCA1" / "sites" / "01-localhost.key" + + backend = StallingTLSBackend(cert, key) + backend.start() + + base_ini = tmp_path / "base.ini" + base_ini.write_text( + f""" +[databases] +stalldb = host=127.0.0.1 port={backend.port} dbname=stalldb + +[pgbouncer] +listen_addr = 127.0.0.1 +auth_type = trust +pool_mode = session +default_pool_size = 5 +server_login_retry = 1 +client_tls_protocols = all +server_tls_protocols = all +server_tls_sslmode = verify-ca +server_tls_ca_file = {ca} +""" + ) + + bouncer = Bouncer(pg, tmp_path / "bouncer", base_ini_path=base_ini) + await bouncer.start() + + cli = None + try: + # Kick off a backend connection; pgBouncer begins the TLS handshake to + # the stalling fake backend. + cli = _trigger_backend_connection(bouncer.host, bouncer.port) + + # Wait until the backend has accepted SSL and parked pgBouncer inside + # the handshake. + assert backend.stalling.wait(timeout=10), ( + "pgBouncer never started the backend TLS handshake" + ) + await asyncio.sleep(0.3) + + # Reload while the handshake is in flight. This frees server_connect_conf. + bouncer.sighup() + + # Let the backend finish the handshake; pgBouncer now reads the TLS + # config it just (in the buggy version) freed. + backend.proceed.set() + await asyncio.sleep(2) + + # pgBouncer must still be alive and responsive. + assert bouncer.running(), ( + "pgBouncer crashed on SIGHUP during a server TLS handshake (issue #1697)" + ) + bouncer.admin("show version") + finally: + if cli is not None: + try: + cli.close() + except OSError: + pass + backend.stop() + await bouncer.cleanup()

view thread (10+ messages)  latest in thread

Message-ID: <gh-pgbouncer-pgbouncer-1698@github.com>
Permalink:  ../gh-pgbouncer-pgbouncer-1698@github.com/

 ·  · 


This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox