agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH v29 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
172+ messages / 2 participants
[nested] [flat]
* [PATCH v29 01/11] Add a syntax to create Incrementally Maintainable Materialized Views
@ 2019-12-20 01:05 Yugo Nagata <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Yugo Nagata @ 2019-12-20 01:05 UTC (permalink / raw)
Allow to create Incrementally Maintainable Materialized View (IMMV)
by using INCREMENTAL option in CREATE MATERIALIZED VIEW command
as follow:
CREATE [INCREMANTAL] MATERIALIZED VIEW xxxxx AS SELECT ....;
---
src/backend/parser/gram.y | 32 +++++++++++++++++++++-----------
src/include/nodes/primnodes.h | 1 +
src/include/parser/kwlist.h | 1 +
3 files changed, 23 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..33c647b0c7 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -465,6 +465,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> OptTempTableName
%type <into> into_clause create_as_target create_mv_target
+%type <boolean> incremental
%type <defelt> createfunc_opt_item common_func_opt_item dostmt_opt_item
%type <fun_param> func_arg func_arg_with_default table_func_column aggr_arg
@@ -718,7 +719,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -4652,32 +4653,34 @@ opt_with_data:
*****************************************************************************/
CreateMatViewStmt:
- CREATE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
+ CREATE OptNoLog incremental MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data
{
CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
- ctas->query = $7;
- ctas->into = $5;
+ ctas->query = $8;
+ ctas->into = $6;
ctas->objtype = OBJECT_MATVIEW;
ctas->is_select_into = false;
ctas->if_not_exists = false;
/* cram additional flags into the IntoClause */
- $5->rel->relpersistence = $2;
- $5->skipData = !($8);
+ $6->rel->relpersistence = $2;
+ $6->skipData = !($9);
+ $6->ivm = $3;
$$ = (Node *) ctas;
}
- | CREATE OptNoLog MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
+ | CREATE OptNoLog incremental MATERIALIZED VIEW IF_P NOT EXISTS create_mv_target AS SelectStmt opt_with_data
{
CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt);
- ctas->query = $10;
- ctas->into = $8;
+ ctas->query = $11;
+ ctas->into = $9;
ctas->objtype = OBJECT_MATVIEW;
ctas->is_select_into = false;
ctas->if_not_exists = true;
/* cram additional flags into the IntoClause */
- $8->rel->relpersistence = $2;
- $8->skipData = !($11);
+ $9->rel->relpersistence = $2;
+ $9->skipData = !($12);
+ $9->ivm = $3;
$$ = (Node *) ctas;
}
;
@@ -4694,9 +4697,14 @@ create_mv_target:
$$->tableSpaceName = $5;
$$->viewQuery = NULL; /* filled at analysis time */
$$->skipData = false; /* might get changed later */
+ $$->ivm = false;
}
;
+incremental: INCREMENTAL { $$ = true; }
+ | /*EMPTY*/ { $$ = false; }
+ ;
+
OptNoLog: UNLOGGED { $$ = RELPERSISTENCE_UNLOGGED; }
| /*EMPTY*/ { $$ = RELPERSISTENCE_PERMANENT; }
;
@@ -17141,6 +17149,7 @@ unreserved_keyword:
| INCLUDE
| INCLUDING
| INCREMENT
+ | INCREMENTAL
| INDENT
| INDEX
| INDEXES
@@ -17709,6 +17718,7 @@ bare_label_keyword:
| INCLUDE
| INCLUDING
| INCREMENT
+ | INCREMENTAL
| INDENT
| INDEX
| INDEXES
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 60d72a876b..cecb968b36 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -146,6 +146,7 @@ typedef struct IntoClause
/* materialized view's SELECT query */
Node *viewQuery pg_node_attr(query_jumble_ignore);
bool skipData; /* true for WITH NO DATA */
+ bool ivm; /* true for WITH IVM */
} IntoClause;
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..d60eb98d65 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -207,6 +207,7 @@ PG_KEYWORD("in", IN_P, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("include", INCLUDE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("including", INCLUDING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("increment", INCREMENT, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("incremental", INCREMENTAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indent", INDENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
--
2.25.1
--Multipart=_Mon__28_Aug_2023_16_05_30_+0900_b1OvQD_3A3ZMTGvj
Content-Type: text/x-diff;
name="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Disposition: attachment;
filename="v29-0002-Add-relisivm-column-to-pg_class-system-catalog.patch"
Content-Transfer-Encoding: 7bit
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
* [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits
@ 2025-03-07 14:44 Andres Freund <[email protected]>
0 siblings, 0 replies; 172+ messages in thread
From: Andres Freund @ 2025-03-07 14:44 UTC (permalink / raw)
The test occasionally failed due to unexpected connection limit errors being
encountered after having waited for FATAL errors on another connection. These
spurious failures were caused by the the backend reporting FATAL errors to the
client before detaching from the PGPROC entry. Adding a sleep(1) before
proc_exit() makes it easy to reproduce that problem.
To fix the issue, add a helper function that waits for postmaster to notice
the process having exited. For now this is implemented by waiting for the
DEBUG2 message that postmaster logs in that case. That's not the prettiest
fix, but simple. If we notice this problem elsewhere, it might be worthwhile
to make this more general, e.g. by adding an injection point.
Reported-by: Tomas Vondra <[email protected]>
Diagnosed-by: Heikki Linnakangas <[email protected]>
Reviewed-by: Heikki Linnakangas <[email protected]>
Discussion: https://postgr.es/m/ggflhkciwdyotpoie323chu2c2idpjk5qimrn462encwx2io7s@thmcxl7i6dpw
---
.../postmaster/t/002_connection_limits.pl | 35 +++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/test/postmaster/t/002_connection_limits.pl b/src/test/postmaster/t/002_connection_limits.pl
index 8cfa6e0ced5..2c185eef6eb 100644
--- a/src/test/postmaster/t/002_connection_limits.pl
+++ b/src/test/postmaster/t/002_connection_limits.pl
@@ -20,6 +20,7 @@ $node->append_conf('postgresql.conf', "max_connections = 6");
$node->append_conf('postgresql.conf', "reserved_connections = 2");
$node->append_conf('postgresql.conf', "superuser_reserved_connections = 1");
$node->append_conf('postgresql.conf', "log_connections = on");
+$node->append_conf('postgresql.conf', "log_min_messages=debug2");
$node->start;
$node->safe_psql(
@@ -45,13 +46,39 @@ sub background_psql_as_user
extra_params => [ '-U', $user ]);
}
+# Like connect_fails(), except that we also wait for the failed backend to
+# have exited.
+#
+# This tests needs to wait for client processes to exit because the error
+# message for a failed connection is reported before the backend has detached
+# from shared memory. If we didn't wait, subsequent tests might hit connection
+# limits spuriously.
+#
+# This can't easily be generalized, as detecting process exit requires
+# log_min_messages to be at least DEBUG2 and is not concurrency safe, as we
+# can't easily be sure the right process exited. In this test that's not a
+# problem though, we only have one new connection at a time.
+sub connect_fails_wait
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($node, $connstr, $test_name, %params) = @_;
+
+ my $log_location = -s $node->logfile;
+
+ $node->connect_fails($connstr, $test_name, %params);
+ $node->wait_for_log(qr/DEBUG: client backend.*exited with exit code 1/,
+ $log_location);
+ ok(1, "$test_name: client backend process exited");
+}
+
my @sessions = ();
my @raw_connections = ();
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
push(@sessions, background_psql_as_user('regress_regular'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -60,7 +87,8 @@ $node->connect_fails(
push(@sessions, background_psql_as_user('regress_reserved'));
push(@sessions, background_psql_as_user('regress_reserved'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_regular",
"reserved_connections limit",
expected_stderr =>
@@ -68,7 +96,8 @@ $node->connect_fails(
);
push(@sessions, background_psql_as_user('regress_superuser'));
-$node->connect_fails(
+connect_fails_wait(
+ $node,
"dbname=postgres user=regress_superuser",
"superuser_reserved_connections limit",
expected_stderr => qr/FATAL: sorry, too many clients already/);
--
2.48.1.76.g4e746b1a31.dirty
--q4d36jgrlhdhudrc
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0004-tests-Don-t-fail-due-to-high-default-timeout-in-p.patch"
^ permalink raw reply [nested|flat] 172+ messages in thread
end of thread, other threads:[~2025-03-07 14:44 UTC | newest]
Thread overview: 172+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-12-20 01:05 [PATCH v29 01/11] Add a syntax to create Incrementally Maintainable Materialized Views Yugo Nagata <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
2025-03-07 14:44 [PATCH v2 3/4] tests: Fix race condition in postmaster/002_connection_limits Andres Freund <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox