Received: from malur.postgresql.org ([217.196.149.56]) by arkaria.postgresql.org with esmtps (TLS1.3:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.92) (envelope-from ) id 1mbd5R-0005Tr-3C for pgsql-committers@arkaria.postgresql.org; Sat, 16 Oct 2021 06:22:30 +0000 Received: from localhost ([127.0.0.1] helo=malur.postgresql.org) by malur.postgresql.org with esmtp (Exim 4.92) (envelope-from ) id 1mbd5P-0003Rh-TC for pgsql-committers@arkaria.postgresql.org; Sat, 16 Oct 2021 06:22:27 +0000 Received: from magus.postgresql.org ([2a02:c0:301:0:ffff::29]) by malur.postgresql.org with esmtps (TLS1.3:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.92) (envelope-from ) id 1mbd5O-0003RY-8J for pgsql-committers@lists.postgresql.org; Sat, 16 Oct 2021 06:22:27 +0000 Received: from wout4-smtp.messagingengine.com ([64.147.123.20]) by magus.postgresql.org with esmtps (TLS1.3:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.92) (envelope-from ) id 1mbd5K-0003B5-Na for pgsql-committers@lists.postgresql.org; Sat, 16 Oct 2021 06:22:25 +0000 Received: from compute1.internal (compute1.nyi.internal [10.202.2.41]) by mailout.west.internal (Postfix) with ESMTP id 4EC46320100E; Sat, 16 Oct 2021 02:22:18 -0400 (EDT) Received: from mailfrontend1 ([10.202.2.162]) by compute1.internal (MEProxy); Sat, 16 Oct 2021 02:22:18 -0400 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=paquier.xyz; h= date:from:to:cc:subject:message-id:references:mime-version :content-type:in-reply-to; s=fm3; bh=3Y48tAlH72wOIvpCMwh6ql+cRcN RSOaEzMaN0iQJhBE=; b=Od1hTXj0TcURMHRGaVvSYsx1FZVeOrLzdjD3zJwQ9zA 8bKGSAHiW1pktDMUP/HNCVxLfcRH1GBYfOFTdBSCt+kWPeadneLwhdR85YT2dcpR A13MCZzVjZvdtBYsMcdwnvZ5M7u2gBPyB/EYJndGvUApfg5eW0k+nDYyQExGDVp9 hTXWGF5+NtonOZsJmiSS+fgm187eseTjjEA2qmAkBxWey1k+Fy1/8nyqo47C1v1I XLfvAl8af5/ZecRmco6GOo8VZgDJ+BHM5TlDIGxUz3JP1KOoSO2ld+Z4506SECci nqxCGBP9CPRVMdBsprxMydxWLZrysaFpvCywdMP7hXQ== DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d= messagingengine.com; h=cc:content-type:date:from:in-reply-to :message-id:mime-version:references:subject:to:x-me-proxy :x-me-proxy:x-me-sender:x-me-sender:x-sasl-enc; s=fm1; bh=3Y48tA lH72wOIvpCMwh6ql+cRcNRSOaEzMaN0iQJhBE=; b=XkWpSB2BMbj0ORhmnJzfv3 BR+0QM/FgtZC5cvuEmPJa+ACm2SBrxvh5xybWvN+BIHOPL4xUQ1HlVdRuyjtG1Pv H/8+9AJ88xy/0+xKVsQBWIPcC6doMmtGLgm/mr4b8IZDp0NYkc9UE+/Z4hxrjI6z 9Jq1C/0E9R/Xf40EFsbm0WGSuTE10tPzViD6SF6LWjqD6gOLk7XDzbj3nUy5begc elboAguKSyOqgITLs/jVZfWDbWCe7bOv1eHmvilNvbLVUpZrLMmM9hQGQXgi22kx YjaKQa+o9TelvaXldLuCmPuK7NDqmgeqv7P6TTkwZlSvHsuTCGLenf3CXzqESvjw == X-ME-Sender: X-ME-Received: X-ME-Proxy-Cause: gggruggvucftvghtrhhoucdtuddrgedvtddrvdduhedguddtfecutefuodetggdotefrod ftvfcurfhrohhfihhlvgemucfhrghsthforghilhdpqfgfvfdpuffrtefokffrpgfnqfgh necuuegrihhlohhuthemuceftddtnecusecvtfgvtghiphhivghnthhsucdlqddutddtmd enfghrlhcuvffnffculdejtddmnecujfgurhepfffhvffukfhfgggtuggjsehgtderredt tddvnecuhfhrohhmpefoihgthhgrvghlucfrrghquhhivghruceomhhitghhrggvlhesph grqhhuihgvrhdrgiihiieqnecuggftrfgrthhtvghrnhepvdegudeuhfdtueeltedtveej heehieevueeigeelteegleejleeiueeiheegvefhnecuvehluhhsthgvrhfuihiivgeptd enucfrrghrrghmpehmrghilhhfrhhomhepmhhitghhrggvlhesphgrqhhuihgvrhdrgiih ii X-ME-Proxy: Received: by mail.messagingengine.com (Postfix) with ESMTPA; Sat, 16 Oct 2021 02:22:16 -0400 (EDT) Date: Sat, 16 Oct 2021 15:22:12 +0900 From: Michael Paquier To: Peter Geoghegan Cc: pgsql-committers , Peter Eisentraut Subject: Re: pgsql: Add more $Test::Builder::Level in the TAP tests Message-ID: References: MIME-Version: 1.0 Content-Type: multipart/signed; micalg=pgp-sha512; protocol="application/pgp-signature"; boundary="OFuFqzKLX8xFC+z0" Content-Disposition: inline In-Reply-To: List-Id: List-Help: List-Subscribe: List-Post: List-Owner: List-Archive: Archived-At: Precedence: bulk --OFuFqzKLX8xFC+z0 Content-Type: multipart/mixed; boundary="RVsFMxMpOjqMeX7n" Content-Disposition: inline --RVsFMxMpOjqMeX7n Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Content-Transfer-Encoding: quoted-printable On Wed, Oct 13, 2021 at 09:04:11PM -0700, Peter Geoghegan wrote: > On Wed, Oct 13, 2021 at 9:00 PM Michael Paquier wro= te: >> It does not change the fact that the test needs to be fixed. TAP >> tests should be designed to not be influenced by the external world. >=20 > Right. Hearing nothing, I have just poked at this problem. And I'd rather make all the tests go through PostgresNode::psql from which it is possible to grab stdout, stderr and the result code, as we could miss local changes in this test if we need a centralized fix, for problems like the one with SIGPIPE where we don't use -c in a psql command. Peter, does it take care of your problem? -- Michael --RVsFMxMpOjqMeX7n Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="psql-tap-fix.patch" Content-Transfer-Encoding: quoted-printable diff --git a/src/bin/psql/t/001_basic.pl b/src/bin/psql/t/001_basic.pl index 685744744a..eb9041d1f3 100644 --- a/src/bin/psql/t/001_basic.pl +++ b/src/bin/psql/t/001_basic.pl @@ -15,10 +15,46 @@ program_options_handling_ok('psql'); my ($stdout, $stderr); my $result; =20 +# Execute a psql command and check its result patterns. +sub psql_like +{ + local $Test::Builder::Level =3D $Test::Builder::Level + 1; + + my $node =3D shift; + my $test_name =3D shift; + my $query =3D shift; + my $expected_stdout =3D shift; + my $expected_stderr =3D shift; + + die "cannot specify both expected stdout and stderr here" + if (defined($expected_stdout) && defined($expected_stderr)); + + # Use the context of a WAL sender, some of the tests rely on that. + my ($ret, $stdout, $stderr) =3D $node->psql( + 'postgres', $query, + on_error_die =3D> 0, + replication =3D> 'database'); + + if (defined($expected_stdout)) + { + is($ret, 0, "$test_name: expected result code"); + is($stderr, '', "$test_name: no stderr"); + like($stdout, $expected_stdout, "$test_name: stdout matches"); + } + if (defined($expected_stderr)) + { + isnt($ret, 0, "$test_name: expected result code"); + like($stderr, $expected_stderr, "$test_name: stderr matches"); + } + + return; +} + # test --help=3Dfoo, analogous to program_help_ok() foreach my $arg (qw(commands variables)) { - $result =3D IPC::Run::run [ 'psql', "--help=3D$arg" ], '>', \$stdout, '2>= ', \$stderr; + $result =3D IPC::Run::run [ 'psql', "--help=3D$arg" ], '>', \$stdout, '2>= ', + \$stderr; ok($result, "psql --help=3D$arg exit code 0"); isnt($stdout, '', "psql --help=3D$arg goes to stdout"); is($stderr, '', "psql --help=3D$arg nothing to stderr"); @@ -34,11 +70,13 @@ max_wal_senders =3D 4 }); $node->start; =20 -$node->command_like([ 'psql', '-c', '\copyright' ], qr/Copyright/, '\copyr= ight'); -$node->command_like([ 'psql', '-c', '\help' ], qr/ALTER/, '\help without a= rguments'); -$node->command_like([ 'psql', '-c', '\help SELECT' ], qr/SELECT/, '\help'); - +psql_like($node, '\copyright', '\copyright', qr/Copyright/, undef); +psql_like($node, '\help without arguments', '\help', qr/ALTER/, undef); +psql_like($node, '\help with argument', '\help SELECT', qr/SELECT/, undef); =20 # Test clean handling of unsupported replication command responses -$node->command_fails_like([ 'psql', '-d', 'replication=3Ddatabase', '-c', = 'START_REPLICATION 0/0' ], - qr/^unexpected PQresultStatus: 8$/, 'handling of unexpected PQresultStatu= s'); +psql_like( + $node, + 'handling of unexpected PQresultStatus', + 'START_REPLICATION 0/0', + undef, qr/unexpected PQresultStatus: 8$/); --RVsFMxMpOjqMeX7n-- --OFuFqzKLX8xFC+z0 Content-Type: application/pgp-signature; name="signature.asc" -----BEGIN PGP SIGNATURE----- iQIzBAABCgAdFiEEG72nH6vTowiyblFKnvQgOdbyQH0FAmFqb5QACgkQnvQgOdby QH074Q//c1lmxFwgXI/Zbrk0N4vOP3CfQIkN10te+ruP/28ApEVpEBjpnn8gLAaI 2U8i+8C9t6yyDb/b4TA+IBbJyeBZCZa9ul7NQbWThkNLLNl4tWBNVmfk9G95zexC cYePYvRmkrxJ4vWvUkKUfkwJBQM/tohTR/ZdRSbIPeyjOc4qAV31o3jhHWrXzvmR 3f1WkUjWYsEYX7EGld9gmMD4KE4K5xO/tLhCyXgEYY48Hovm7vTGd+i7mn03fX6p elZVSyth8l6+r+nJ+DzjjxWVBhZCbVfRISKBVe+J9ZCRdPQJz+QO1821LQEaetSm BiYTIzdcIV509qyB1h64R9IjtHpVSdE8K5Zf9sd2Uu0hhqsNROOhH7TjsCk+iYHE kcaJypgw8Bt/MIOF2/WOgtqO+E7hWrqN1FRMYs9JdOnyhEA+Mkpnkkc5KZXIuIj5 34aa4PDR7cPWiV0FuYtcZK7HPSpW/y/LdoJ9Zro/+bSq8NqkgI0jF/5qBea9ilPG s+8V2dfvl/OkTbhekvWId4J+juygZcq1Mt+PannA1fTFk60WW/7n/KZ47DaZG9xd RMswQukxtoe5eWuJ6+7xmDDopfcjJLTjCwtdJjj5bh8m0QHqnz0+QCsnmhaVdmXi ytkvOjTfMY7O+R7lyoy+tfCden4sekkw4r1++tbh6FJL54GqLqA= =62Zt -----END PGP SIGNATURE----- --OFuFqzKLX8xFC+z0--