Date: Wed, 19 Mar 2025 16:28:23 -0500
From: Nathan Bossart <nathandbossart@gmail.com>
To: Andres Freund <andres@anarazel.de>
Cc: Tom Lane <tgl@sss.pgh.pa.us>, Robert Haas <robertmhaas@gmail.com>,
	Greg Sabino Mullane <htamfids@gmail.com>,
	pgsql-hackers@postgresql.org, bruce@momjian.us
Subject: Re: optimize file transfer in pg_upgrade
Message-ID: <Z9s29-mO6wnSEU_r@nathan>
References: <Z9mtXsBs-31E5ksW@nathan>
 <tlkgkf3woe4ye2l4m4dnwui4wmqtajotozw5r3fhefyuv6hwrl@34xroqbvddpj>
 <Z9mxlf5uVuk2DziP@nathan>
 <7rhxl5nl5cuenm4kzprjhaojo74m2teuwzjffcrdi6bxx6b2wn@orb5h3mteh2y>
 <Z9nEup6k56N_9l0X@nathan>
 <Z9oofphRMQQzgRmL@nathan>
 <Z9rjSmA_MpPz-zOw@nathan>
 <712605.1742401713@sss.pgh.pa.us>
 <can7m7lntp3rpwchr7upnlom4fhzwfwttfa53xks2yxxafxfkv@3nzrihnqmx4g>
 <Z9sbsU4cu7O9glEQ@nathan>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="C5ohSC+cpjxht8ja"
Content-Disposition: inline
In-Reply-To: <Z9sbsU4cu7O9glEQ@nathan>
Archived-At: <https://www.postgresql.org/message-id/Z9s29-mO6wnSEU_r%40nathan>
Precedence: bulk


--C5ohSC+cpjxht8ja
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

On Wed, Mar 19, 2025 at 02:32:01PM -0500, Nathan Bossart wrote:
> In addition to testing with in-place tablespaces, we might also want to
> teach the transfer modes test to do cross-version testing when possible.
> In that case, we can test normal (non-in-place) tablespaces.  However, that
> would be limited to the buildfarm.

Actually, this one was pretty easy to do.

-- 
nathan

--C5ohSC+cpjxht8ja
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
	filename="0001-Add-test-for-pg_upgrade-file-transfer-modes.patch"

From cc9bcd456f0d7d6ab19a89813755a3e76993cfb9 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Tue, 18 Mar 2025 20:58:07 -0500
Subject: [PATCH 1/1] Add test for pg_upgrade file transfer modes.

This new test checks all of pg_upgrade's file transfer modes.  For
each mode, we verify that pg_upgrade either succeeds (and some test
objects successfully reach the new version) or fails with an error
that indicates the mode is not supported on the current platform.

Suggested-by: Robert Haas <robertmhaas@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/Zyvop-LxLXBLrZil%40nathan
---
 src/bin/pg_upgrade/meson.build           |  1 +
 src/bin/pg_upgrade/t/006_modes.pl        | 89 ++++++++++++++++++++++++
 src/test/perl/PostgreSQL/Test/Cluster.pm | 19 +++++
 src/test/perl/PostgreSQL/Test/Utils.pm   | 25 +++++++
 4 files changed, 134 insertions(+)
 create mode 100644 src/bin/pg_upgrade/t/006_modes.pl

diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build
index da84344966a..16cd9247e76 100644
--- a/src/bin/pg_upgrade/meson.build
+++ b/src/bin/pg_upgrade/meson.build
@@ -46,6 +46,7 @@ tests += {
       't/003_logical_slots.pl',
       't/004_subscription.pl',
       't/005_char_signedness.pl',
+      't/006_modes.pl',
     ],
     'test_kwargs': {'priority': 40}, # pg_upgrade tests are slow
   },
diff --git a/src/bin/pg_upgrade/t/006_modes.pl b/src/bin/pg_upgrade/t/006_modes.pl
new file mode 100644
index 00000000000..85098e86245
--- /dev/null
+++ b/src/bin/pg_upgrade/t/006_modes.pl
@@ -0,0 +1,89 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+# Tests for file transfer modes
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_mode
+{
+	my ($mode) = @_;
+
+	my $old = PostgreSQL::Test::Cluster->new('old', install_path => $ENV{oldinstall});
+	my $new = PostgreSQL::Test::Cluster->new('new');
+
+	if (defined($ENV{oldinstall}))
+	{
+		$old->init(force_initdb => 1, extra => ['-k']);
+	}
+	else
+	{
+		$old->init();
+	}
+	$new->init();
+
+	$old->start;
+	$old->safe_psql('postgres', "CREATE TABLE test1 AS SELECT generate_series(1, 100)");
+	$old->safe_psql('postgres', "CREATE DATABASE testdb1");
+	$old->safe_psql('testdb1', "CREATE TABLE test2 AS SELECT generate_series(200, 300)");
+	$old->safe_psql('testdb1', "VACUUM FULL test2");
+	$old->safe_psql('testdb1', "CREATE SEQUENCE testseq START 5432");
+	if (defined($ENV{oldinstall}))
+	{
+		my $tblspc = PostgreSQL::Test::Utils::tempdir_short();
+		$old->safe_psql('postgres', "CREATE TABLESPACE test_tblspc LOCATION '$tblspc'");
+		$old->safe_psql('postgres', "CREATE DATABASE testdb2 TABLESPACE test_tblspc");
+		$old->safe_psql('postgres', "CREATE TABLE test3 TABLESPACE test_tblspc AS SELECT generate_series(300, 401)");
+		$old->safe_psql('testdb2', "CREATE TABLE test4 AS SELECT generate_series(400, 502)");
+	}
+	$old->stop;
+
+	my $result = command_ok_or_fails_like(
+		[
+			'pg_upgrade', '--no-sync',
+			'--old-datadir' => $old->data_dir,
+			'--new-datadir' => $new->data_dir,
+			'--old-bindir' => $old->config_data('--bindir'),
+			'--new-bindir' => $new->config_data('--bindir'),
+			'--socketdir' => $new->host,
+			'--old-port' => $old->port,
+			'--new-port' => $new->port,
+			$mode
+		],
+		qr/.* not supported on this platform|could not .* between old and new data directories: .*/,
+		qr/^$/,
+		"pg_upgrade with transfer mode $mode");
+
+	if ($result)
+	{
+		$new->start;
+		$result = $new->safe_psql('postgres', "SELECT COUNT(*) FROM test1");
+		is($result, '100', "test1 data after pg_upgrade $mode");
+		$result = $new->safe_psql('testdb1', "SELECT COUNT(*) FROM test2");
+		is($result, '101', "test2 data after pg_upgrade $mode");
+		$result = $new->safe_psql('testdb1', "SELECT nextval('testseq')");
+		is($result, '5432', "sequence data after pg_upgrade $mode");
+		if (defined($ENV{oldinstall}))
+		{
+			$result = $new->safe_psql('postgres', "SELECT COUNT(*) FROM test3");
+			is($result, '102', "test3 data after pg_upgrade $mode");
+			$result = $new->safe_psql('testdb2', "SELECT COUNT(*) FROM test4");
+			is($result, '103', "test4 data after pg_upgrade $mode");
+		}
+		$new->stop;
+	}
+
+	$old->clean_node();
+	$new->clean_node();
+}
+
+test_mode('--clone');
+test_mode('--copy');
+test_mode('--copy-file-range');
+test_mode('--link');
+
+done_testing();
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 05bd94609d4..8759ed2cbba 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2801,6 +2801,25 @@ sub command_fails_like
 
 =pod
 
+=item $node->command_ok_or_fails_like(...)
+
+PostgreSQL::Test::Utils::command_ok_or_fails_like with our connection parameters. See command_ok(...)
+
+=cut
+
+sub command_ok_or_fails_like
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+	my $self = shift;
+
+	local %ENV = $self->_get_env();
+
+	return PostgreSQL::Test::Utils::command_ok_or_fails_like(@_);
+}
+
+=pod
+
 =item $node->command_checks_all(...)
 
 PostgreSQL::Test::Utils::command_checks_all with our connection parameters. See
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index d1ad131eadf..7d7ca83495f 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -89,6 +89,7 @@ our @EXPORT = qw(
   command_like
   command_like_safe
   command_fails_like
+  command_ok_or_fails_like
   command_checks_all
 
   $windows_os
@@ -1067,6 +1068,30 @@ sub command_fails_like
 
 =pod
 
+=item command_ok_or_fails_like(cmd, expected_stdout, expected_stderr, test_name)
+
+Check that the command either succeeds or fails with an error that matches the
+given regular expressions.
+
+=cut
+
+sub command_ok_or_fails_like
+{
+	local $Test::Builder::Level = $Test::Builder::Level + 1;
+	my ($cmd, $expected_stdout, $expected_stderr, $test_name) = @_;
+	my ($stdout, $stderr);
+	print("# Running: " . join(" ", @{$cmd}) . "\n");
+	my $result = IPC::Run::run $cmd, '>' => \$stdout, '2>' => \$stderr;
+	if (!$result)
+	{
+		like($stdout, $expected_stdout, "$test_name: stdout matches");
+		like($stderr, $expected_stderr, "$test_name: stderr matches");
+	}
+	return $result;
+}
+
+=pod
+
 =item command_checks_all(cmd, ret, out, err, test_name)
 
 Run a command and check its status and outputs.
-- 
2.39.5 (Apple Git-154)


--C5ohSC+cpjxht8ja--