pg.ddx.io pgsql-hackers@postgresql.org mailing list archive help / color / mirror / Atom feed From: Nathan Bossart <nathandbossart@gmail.com>
To: Andres Freund <andres@anarazel.de>
Cc: Tom Lane <tgl@sss.pgh.pa.us>
Cc: Robert Haas <robertmhaas@gmail.com>
Cc: Greg Sabino Mullane <htamfids@gmail.com>
Cc: pgsql-hackers@postgresql.org
Cc: bruce@momjian.us
Subject: Re: optimize file transfer in pg_upgrade
Date: Wed, 19 Mar 2025 16:28:23 -0500
Message-ID: <Z9s29-mO6wnSEU_r@nathan> (raw )
In-Reply-To: <Z9sbsU4cu7O9glEQ@nathan >
References: <Z9mtXsBs-31E5ksW@nathan >
<tlkgkf3woe4ye2l4m4dnwui4wmqtajotozw5r3fhefyuv6hwrl@34xroqbvddpj >
<Z9mxlf5uVuk2DziP@nathan >
<7rhxl5nl5cuenm4kzprjhaojo74m2teuwzjffcrdi6bxx6b2wn@orb5h3mteh2y >
<Z9nEup6k56N_9l0X@nathan >
<Z9oofphRMQQzgRmL@nathan >
<Z9rjSmA_MpPz-zOw@nathan >
<712605.1742401713@sss.pgh.pa.us >
<can7m7lntp3rpwchr7upnlom4fhzwfwttfa53xks2yxxafxfkv@3nzrihnqmx4g >
<Z9sbsU4cu7O9glEQ@nathan >
On Wed, Mar 19, 2025 at 02:32:01PM -0500, Nathan Bossart wrote:
> In addition to testing with in-place tablespaces, we might also want to
> teach the transfer modes test to do cross-version testing when possible.
> In that case, we can test normal (non-in-place) tablespaces. However, that
> would be limited to the buildfarm.
Actually, this one was pretty easy to do.
--
nathan
From cc9bcd456f0d7d6ab19a89813755a3e76993cfb9 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Tue, 18 Mar 2025 20:58:07 -0500
Subject: [PATCH 1/1] Add test for pg_upgrade file transfer modes.
This new test checks all of pg_upgrade's file transfer modes. For
each mode, we verify that pg_upgrade either succeeds (and some test
objects successfully reach the new version) or fails with an error
that indicates the mode is not supported on the current platform.
Suggested-by: Robert Haas <robertmhaas@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/Zyvop-LxLXBLrZil%40nathan
---
src/bin/pg_upgrade/meson.build | 1 +
src/bin/pg_upgrade/t/006_modes.pl | 89 ++++++++++++++++++++++++
src/test/perl/PostgreSQL/Test/Cluster.pm | 19 +++++
src/test/perl/PostgreSQL/Test/Utils.pm | 25 +++++++
4 files changed, 134 insertions(+)
create mode 100644 src/bin/pg_upgrade/t/006_modes.pl
diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build
index da84344966a..16cd9247e76 100644
--- a/src/bin/pg_upgrade/meson.build
+++ b/src/bin/pg_upgrade/meson.build
@@ -46,6 +46,7 @@ tests += {
't/003_logical_slots.pl',
't/004_subscription.pl',
't/005_char_signedness.pl',
+ 't/006_modes.pl',
],
'test_kwargs': {'priority': 40}, # pg_upgrade tests are slow
},
diff --git a/src/bin/pg_upgrade/t/006_modes.pl b/src/bin/pg_upgrade/t/006_modes.pl
new file mode 100644
index 00000000000..85098e86245
--- /dev/null
+++ b/src/bin/pg_upgrade/t/006_modes.pl
@@ -0,0 +1,89 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+# Tests for file transfer modes
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_mode
+{
+ my ($mode) = @_;
+
+ my $old = PostgreSQL::Test::Cluster->new('old', install_path => $ENV{oldinstall});
+ my $new = PostgreSQL::Test::Cluster->new('new');
+
+ if (defined($ENV{oldinstall}))
+ {
+ $old->init(force_initdb => 1, extra => ['-k']);
+ }
+ else
+ {
+ $old->init();
+ }
+ $new->init();
+
+ $old->start;
+ $old->safe_psql('postgres', "CREATE TABLE test1 AS SELECT generate_series(1, 100)");
+ $old->safe_psql('postgres', "CREATE DATABASE testdb1");
+ $old->safe_psql('testdb1', "CREATE TABLE test2 AS SELECT generate_series(200, 300)");
+ $old->safe_psql('testdb1', "VACUUM FULL test2");
+ $old->safe_psql('testdb1', "CREATE SEQUENCE testseq START 5432");
+ if (defined($ENV{oldinstall}))
+ {
+ my $tblspc = PostgreSQL::Test::Utils::tempdir_short();
+ $old->safe_psql('postgres', "CREATE TABLESPACE test_tblspc LOCATION '$tblspc'");
+ $old->safe_psql('postgres', "CREATE DATABASE testdb2 TABLESPACE test_tblspc");
+ $old->safe_psql('postgres', "CREATE TABLE test3 TABLESPACE test_tblspc AS SELECT generate_series(300, 401)");
+ $old->safe_psql('testdb2', "CREATE TABLE test4 AS SELECT generate_series(400, 502)");
+ }
+ $old->stop;
+
+ my $result = command_ok_or_fails_like(
+ [
+ 'pg_upgrade', '--no-sync',
+ '--old-datadir' => $old->data_dir,
+ '--new-datadir' => $new->data_dir,
+ '--old-bindir' => $old->config_data('--bindir'),
+ '--new-bindir' => $new->config_data('--bindir'),
+ '--socketdir' => $new->host,
+ '--old-port' => $old->port,
+ '--new-port' => $new->port,
+ $mode
+ ],
+ qr/.* not supported on this platform|could not .* between old and new data directories: .*/,
+ qr/^$/,
+ "pg_upgrade with transfer mode $mode");
+
+ if ($result)
+ {
+ $new->start;
+ $result = $new->safe_psql('postgres', "SELECT COUNT(*) FROM test1");
+ is($result, '100', "test1 data after pg_upgrade $mode");
+ $result = $new->safe_psql('testdb1', "SELECT COUNT(*) FROM test2");
+ is($result, '101', "test2 data after pg_upgrade $mode");
+ $result = $new->safe_psql('testdb1', "SELECT nextval('testseq')");
+ is($result, '5432', "sequence data after pg_upgrade $mode");
+ if (defined($ENV{oldinstall}))
+ {
+ $result = $new->safe_psql('postgres', "SELECT COUNT(*) FROM test3");
+ is($result, '102', "test3 data after pg_upgrade $mode");
+ $result = $new->safe_psql('testdb2', "SELECT COUNT(*) FROM test4");
+ is($result, '103', "test4 data after pg_upgrade $mode");
+ }
+ $new->stop;
+ }
+
+ $old->clean_node();
+ $new->clean_node();
+}
+
+test_mode('--clone');
+test_mode('--copy');
+test_mode('--copy-file-range');
+test_mode('--link');
+
+done_testing();
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 05bd94609d4..8759ed2cbba 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2801,6 +2801,25 @@ sub command_fails_like
=pod
+=item $node->command_ok_or_fails_like(...)
+
+PostgreSQL::Test::Utils::command_ok_or_fails_like with our connection parameters. See command_ok(...)
+
+=cut
+
+sub command_ok_or_fails_like
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my $self = shift;
+
+ local %ENV = $self->_get_env();
+
+ return PostgreSQL::Test::Utils::command_ok_or_fails_like(@_);
+}
+
+=pod
+
=item $node->command_checks_all(...)
PostgreSQL::Test::Utils::command_checks_all with our connection parameters. See
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index d1ad131eadf..7d7ca83495f 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -89,6 +89,7 @@ our @EXPORT = qw(
command_like
command_like_safe
command_fails_like
+ command_ok_or_fails_like
command_checks_all
$windows_os
@@ -1067,6 +1068,30 @@ sub command_fails_like
=pod
+=item command_ok_or_fails_like(cmd, expected_stdout, expected_stderr, test_name)
+
+Check that the command either succeeds or fails with an error that matches the
+given regular expressions.
+
+=cut
+
+sub command_ok_or_fails_like
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($cmd, $expected_stdout, $expected_stderr, $test_name) = @_;
+ my ($stdout, $stderr);
+ print("# Running: " . join(" ", @{$cmd}) . "\n");
+ my $result = IPC::Run::run $cmd, '>' => \$stdout, '2>' => \$stderr;
+ if (!$result)
+ {
+ like($stdout, $expected_stdout, "$test_name: stdout matches");
+ like($stderr, $expected_stderr, "$test_name: stderr matches");
+ }
+ return $result;
+}
+
+=pod
+
=item command_checks_all(cmd, ret, out, err, test_name)
Run a command and check its status and outputs.
--
2.39.5 (Apple Git-154)
Attachments:
[text/plain] 0001-Add-test-for-pg_upgrade-file-transfer-modes.patch (6.4K, ../Z9s29-mO6wnSEU_r@nathan/2-0001-Add-test-for-pg_upgrade-file-transfer-modes.patch)
download | inline diff:
From cc9bcd456f0d7d6ab19a89813755a3e76993cfb9 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Tue, 18 Mar 2025 20:58:07 -0500
Subject: [PATCH 1/1] Add test for pg_upgrade file transfer modes.
This new test checks all of pg_upgrade's file transfer modes. For
each mode, we verify that pg_upgrade either succeeds (and some test
objects successfully reach the new version) or fails with an error
that indicates the mode is not supported on the current platform.
Suggested-by: Robert Haas <robertmhaas@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/Zyvop-LxLXBLrZil%40nathan
---
src/bin/pg_upgrade/meson.build | 1 +
src/bin/pg_upgrade/t/006_modes.pl | 89 ++++++++++++++++++++++++
src/test/perl/PostgreSQL/Test/Cluster.pm | 19 +++++
src/test/perl/PostgreSQL/Test/Utils.pm | 25 +++++++
4 files changed, 134 insertions(+)
create mode 100644 src/bin/pg_upgrade/t/006_modes.pl
diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build
index da84344966a..16cd9247e76 100644
--- a/src/bin/pg_upgrade/meson.build
+++ b/src/bin/pg_upgrade/meson.build
@@ -46,6 +46,7 @@ tests += {
't/003_logical_slots.pl',
't/004_subscription.pl',
't/005_char_signedness.pl',
+ 't/006_modes.pl',
],
'test_kwargs': {'priority': 40}, # pg_upgrade tests are slow
},
diff --git a/src/bin/pg_upgrade/t/006_modes.pl b/src/bin/pg_upgrade/t/006_modes.pl
new file mode 100644
index 00000000000..85098e86245
--- /dev/null
+++ b/src/bin/pg_upgrade/t/006_modes.pl
@@ -0,0 +1,89 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+# Tests for file transfer modes
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+sub test_mode
+{
+ my ($mode) = @_;
+
+ my $old = PostgreSQL::Test::Cluster->new('old', install_path => $ENV{oldinstall});
+ my $new = PostgreSQL::Test::Cluster->new('new');
+
+ if (defined($ENV{oldinstall}))
+ {
+ $old->init(force_initdb => 1, extra => ['-k']);
+ }
+ else
+ {
+ $old->init();
+ }
+ $new->init();
+
+ $old->start;
+ $old->safe_psql('postgres', "CREATE TABLE test1 AS SELECT generate_series(1, 100)");
+ $old->safe_psql('postgres', "CREATE DATABASE testdb1");
+ $old->safe_psql('testdb1', "CREATE TABLE test2 AS SELECT generate_series(200, 300)");
+ $old->safe_psql('testdb1', "VACUUM FULL test2");
+ $old->safe_psql('testdb1', "CREATE SEQUENCE testseq START 5432");
+ if (defined($ENV{oldinstall}))
+ {
+ my $tblspc = PostgreSQL::Test::Utils::tempdir_short();
+ $old->safe_psql('postgres', "CREATE TABLESPACE test_tblspc LOCATION '$tblspc'");
+ $old->safe_psql('postgres', "CREATE DATABASE testdb2 TABLESPACE test_tblspc");
+ $old->safe_psql('postgres', "CREATE TABLE test3 TABLESPACE test_tblspc AS SELECT generate_series(300, 401)");
+ $old->safe_psql('testdb2', "CREATE TABLE test4 AS SELECT generate_series(400, 502)");
+ }
+ $old->stop;
+
+ my $result = command_ok_or_fails_like(
+ [
+ 'pg_upgrade', '--no-sync',
+ '--old-datadir' => $old->data_dir,
+ '--new-datadir' => $new->data_dir,
+ '--old-bindir' => $old->config_data('--bindir'),
+ '--new-bindir' => $new->config_data('--bindir'),
+ '--socketdir' => $new->host,
+ '--old-port' => $old->port,
+ '--new-port' => $new->port,
+ $mode
+ ],
+ qr/.* not supported on this platform|could not .* between old and new data directories: .*/,
+ qr/^$/,
+ "pg_upgrade with transfer mode $mode");
+
+ if ($result)
+ {
+ $new->start;
+ $result = $new->safe_psql('postgres', "SELECT COUNT(*) FROM test1");
+ is($result, '100', "test1 data after pg_upgrade $mode");
+ $result = $new->safe_psql('testdb1', "SELECT COUNT(*) FROM test2");
+ is($result, '101', "test2 data after pg_upgrade $mode");
+ $result = $new->safe_psql('testdb1', "SELECT nextval('testseq')");
+ is($result, '5432', "sequence data after pg_upgrade $mode");
+ if (defined($ENV{oldinstall}))
+ {
+ $result = $new->safe_psql('postgres', "SELECT COUNT(*) FROM test3");
+ is($result, '102', "test3 data after pg_upgrade $mode");
+ $result = $new->safe_psql('testdb2', "SELECT COUNT(*) FROM test4");
+ is($result, '103', "test4 data after pg_upgrade $mode");
+ }
+ $new->stop;
+ }
+
+ $old->clean_node();
+ $new->clean_node();
+}
+
+test_mode('--clone');
+test_mode('--copy');
+test_mode('--copy-file-range');
+test_mode('--link');
+
+done_testing();
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 05bd94609d4..8759ed2cbba 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2801,6 +2801,25 @@ sub command_fails_like
=pod
+=item $node->command_ok_or_fails_like(...)
+
+PostgreSQL::Test::Utils::command_ok_or_fails_like with our connection parameters. See command_ok(...)
+
+=cut
+
+sub command_ok_or_fails_like
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my $self = shift;
+
+ local %ENV = $self->_get_env();
+
+ return PostgreSQL::Test::Utils::command_ok_or_fails_like(@_);
+}
+
+=pod
+
=item $node->command_checks_all(...)
PostgreSQL::Test::Utils::command_checks_all with our connection parameters. See
diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm b/src/test/perl/PostgreSQL/Test/Utils.pm
index d1ad131eadf..7d7ca83495f 100644
--- a/src/test/perl/PostgreSQL/Test/Utils.pm
+++ b/src/test/perl/PostgreSQL/Test/Utils.pm
@@ -89,6 +89,7 @@ our @EXPORT = qw(
command_like
command_like_safe
command_fails_like
+ command_ok_or_fails_like
command_checks_all
$windows_os
@@ -1067,6 +1068,30 @@ sub command_fails_like
=pod
+=item command_ok_or_fails_like(cmd, expected_stdout, expected_stderr, test_name)
+
+Check that the command either succeeds or fails with an error that matches the
+given regular expressions.
+
+=cut
+
+sub command_ok_or_fails_like
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+ my ($cmd, $expected_stdout, $expected_stderr, $test_name) = @_;
+ my ($stdout, $stderr);
+ print("# Running: " . join(" ", @{$cmd}) . "\n");
+ my $result = IPC::Run::run $cmd, '>' => \$stdout, '2>' => \$stderr;
+ if (!$result)
+ {
+ like($stdout, $expected_stdout, "$test_name: stdout matches");
+ like($stderr, $expected_stderr, "$test_name: stderr matches");
+ }
+ return $result;
+}
+
+=pod
+
=item command_checks_all(cmd, ret, out, err, test_name)
Run a command and check its status and outputs.
--
2.39.5 (Apple Git-154)
view thread (45+ messages) latest in thread
Message-ID: <Z9s29-mO6wnSEU_r@nathan>
Permalink: ../Z9s29-mO6wnSEU_r@nathan/
Also on: postgresql.org/message-id/Z9s29-mO6wnSEU_r@nathan
copy link · copy postgr.es · copy git am
reply Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: nathandbossart@gmail.com, andres@anarazel.de, tgl@sss.pgh.pa.us, robertmhaas@gmail.com, htamfids@gmail.com, bruce@momjian.us
Subject: Re: optimize file transfer in pg_upgrade
In-Reply-To: <Z9s29-mO6wnSEU_r@nathan>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox