public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v2 1/1] resownerbench
27+ messages / 8 participants
[nested] [flat]
* [PATCH v2 1/1] resownerbench
@ 2020-11-11 22:40 Heikki Linnakangas <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Heikki Linnakangas @ 2020-11-11 22:40 UTC (permalink / raw)
---
contrib/resownerbench/Makefile | 17 ++
contrib/resownerbench/resownerbench--1.0.sql | 14 ++
contrib/resownerbench/resownerbench.c | 154 +++++++++++++++++++
contrib/resownerbench/resownerbench.control | 6 +
contrib/resownerbench/snaptest.sql | 37 +++++
5 files changed, 228 insertions(+)
create mode 100644 contrib/resownerbench/Makefile
create mode 100644 contrib/resownerbench/resownerbench--1.0.sql
create mode 100644 contrib/resownerbench/resownerbench.c
create mode 100644 contrib/resownerbench/resownerbench.control
create mode 100644 contrib/resownerbench/snaptest.sql
diff --git a/contrib/resownerbench/Makefile b/contrib/resownerbench/Makefile
new file mode 100644
index 00000000000..9b0e1cfee1a
--- /dev/null
+++ b/contrib/resownerbench/Makefile
@@ -0,0 +1,17 @@
+MODULE_big = resownerbench
+OBJS = resownerbench.o
+
+EXTENSION = resownerbench
+DATA = resownerbench--1.0.sql
+PGFILEDESC = "resownerbench - benchmark for ResourceOwners"
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/resownerbench
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/contrib/resownerbench/resownerbench--1.0.sql b/contrib/resownerbench/resownerbench--1.0.sql
new file mode 100644
index 00000000000..d29182f5982
--- /dev/null
+++ b/contrib/resownerbench/resownerbench--1.0.sql
@@ -0,0 +1,14 @@
+/* contrib/resownerbench/resownerbench--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION resownerbench" to load this file. \quit
+
+CREATE FUNCTION snapshotbench_lifo(numkeep int, numsnaps int, numiters int)
+RETURNS double precision
+AS 'MODULE_PATHNAME', 'snapshotbench_lifo'
+LANGUAGE C STRICT VOLATILE;
+
+CREATE FUNCTION snapshotbench_fifo(numkeep int, numsnaps int, numiters int)
+RETURNS double precision
+AS 'MODULE_PATHNAME', 'snapshotbench_fifo'
+LANGUAGE C STRICT VOLATILE;
diff --git a/contrib/resownerbench/resownerbench.c b/contrib/resownerbench/resownerbench.c
new file mode 100644
index 00000000000..acfb6c39199
--- /dev/null
+++ b/contrib/resownerbench/resownerbench.c
@@ -0,0 +1,154 @@
+#include "postgres.h"
+
+#include "catalog/pg_type.h"
+#include "catalog/pg_statistic.h"
+#include "executor/spi.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "utils/catcache.h"
+#include "utils/syscache.h"
+#include "utils/timestamp.h"
+#include "utils/snapmgr.h"
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(snapshotbench_lifo);
+PG_FUNCTION_INFO_V1(snapshotbench_fifo);
+
+/*
+ * ResourceOwner Performance test, using RegisterSnapshot().
+ *
+ * This takes three parameters: numkeep, numsnaps, numiters.
+ *
+ * First, we register 'numkeep' snapshots. They are kept registed
+ * until the end of the test. Then, we repeatedly register and
+ * unregister 'numsnaps - numkeep' additional snapshots, repeating
+ * 'numiters' times. All the register/unregister calls are made in
+ * LIFO order.
+ *
+ * Returns the time spent, in milliseconds.
+ *
+ * The idea is to test the performance of ResourceOwnerRemember()
+ * and ReourceOwnerForget() operations, under different regimes.
+ *
+ * In the old implementation, if 'numsnaps' is small enough, all
+ * the entries fit in the resource owner's small array (it can
+ * hold 64 entries).
+ *
+ * In the new implementation, the array is much smaller, only 8
+ * entries, but it's used together with the hash table so that
+ * we stay in the "array regime" as long as 'numsnaps - numkeep'
+ * is smaller than 8 entries.
+ *
+ * 'numiters' can be adjusted to adjust the overall runtime to be
+ * suitable long.
+ */
+Datum
+snapshotbench_lifo(PG_FUNCTION_ARGS)
+{
+ int numkeep = PG_GETARG_INT32(0);
+ int numsnaps = PG_GETARG_INT32(1);
+ int numiters = PG_GETARG_INT32(2);
+ int i;
+ instr_time start,
+ duration;
+ Snapshot lsnap;
+ Snapshot *rs;
+ int numregistered = 0;
+
+ rs = palloc(Max(numsnaps, numkeep) * sizeof(Snapshot));
+
+ lsnap = GetLatestSnapshot();
+
+ PG_SETMASK(&BlockSig);
+ INSTR_TIME_SET_CURRENT(start);
+
+ while (numregistered < numkeep)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ for (i = 0 ; i < numiters; i++)
+ {
+ while (numregistered < numsnaps)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ while (numregistered > numkeep)
+ {
+ numregistered--;
+ UnregisterSnapshot(rs[numregistered]);
+ }
+ }
+
+ while (numregistered > 0)
+ {
+ numregistered--;
+ UnregisterSnapshot(rs[numregistered]);
+ }
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ PG_SETMASK(&UnBlockSig);
+
+ PG_RETURN_FLOAT8(INSTR_TIME_GET_MILLISEC(duration));
+};
+
+
+/*
+ * Same, but do the register/unregister operations in
+ * FIFO order.
+ */
+Datum
+snapshotbench_fifo(PG_FUNCTION_ARGS)
+{
+ int numkeep = PG_GETARG_INT32(0);
+ int numsnaps = PG_GETARG_INT32(1);
+ int numiters = PG_GETARG_INT32(2);
+ int i,
+ j;
+ instr_time start,
+ duration;
+ Snapshot lsnap;
+ Snapshot *rs;
+ int numregistered = 0;
+
+ rs = palloc(Max(numsnaps, numkeep) * sizeof(Snapshot));
+
+ lsnap = GetLatestSnapshot();
+
+ PG_SETMASK(&BlockSig);
+ INSTR_TIME_SET_CURRENT(start);
+
+ while (numregistered < numkeep)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ for (i = 0 ; i < numiters; i++)
+ {
+ while (numregistered < numsnaps)
+ {
+ rs[numregistered] = RegisterSnapshot(lsnap);
+ numregistered++;
+ }
+
+ for (j = numkeep; j < numregistered; j++)
+ UnregisterSnapshot(rs[j]);
+ numregistered = numkeep;
+ }
+
+ for (j = 0; j < numregistered; j++)
+ UnregisterSnapshot(rs[j]);
+ numregistered = numkeep;
+
+ INSTR_TIME_SET_CURRENT(duration);
+ INSTR_TIME_SUBTRACT(duration, start);
+ PG_SETMASK(&UnBlockSig);
+
+ PG_RETURN_FLOAT8(INSTR_TIME_GET_MILLISEC(duration));
+};
diff --git a/contrib/resownerbench/resownerbench.control b/contrib/resownerbench/resownerbench.control
new file mode 100644
index 00000000000..ada88b8eed8
--- /dev/null
+++ b/contrib/resownerbench/resownerbench.control
@@ -0,0 +1,6 @@
+# resownerbench
+
+comment = 'benchmark for ResourceOwners'
+default_version = '1.0'
+module_pathname = '$libdir/resownerbench'
+relocatable = true
diff --git a/contrib/resownerbench/snaptest.sql b/contrib/resownerbench/snaptest.sql
new file mode 100644
index 00000000000..e3c0a9710ee
--- /dev/null
+++ b/contrib/resownerbench/snaptest.sql
@@ -0,0 +1,37 @@
+--
+-- Performance test RegisterSnapshot/UnregisterSnapshot.
+--
+select numkeep, numsnaps,
+ -- numiters,
+ -- round(lifo_time_ms) as lifo_total_time_ms,
+ -- round(fifo_time_ms) as fifo_total_time_ms,
+ round((lifo_time_ms::numeric / (numkeep + (numsnaps - numkeep) * numiters)) * 1000000, 1) as lifo_time_ns,
+ round((fifo_time_ms::numeric / (numkeep + (numsnaps - numkeep) * numiters)) * 1000000, 1) as fifo_time_ns
+from
+(values (0, 1, 10000000 * 10),
+ (0, 5, 2000000 * 10),
+ (0, 10, 1000000 * 10),
+ (0, 60, 100000 * 10),
+ (0, 70, 100000 * 10),
+ (0, 100, 100000 * 10),
+ (0, 1000, 10000 * 10),
+ (0, 10000, 1000 * 10),
+
+-- These tests keep 9 snapshots registered across the iterations. That
+-- exceeds the size of the little array in the patch, so this exercises
+-- the hash lookups. Without the patch, these still fit in the array
+-- (it's 64 entries without the patch)
+ (9, 10, 10000000 * 10),
+ (9, 100, 100000 * 10),
+ (9, 1000, 10000 * 10),
+ (9, 10000, 1000 * 10),
+
+-- These exceed the 64 entry array even without the patch, so these fall
+-- in the hash table regime with and without the patch.
+ (65, 70, 1000000 * 10),
+ (65, 100, 100000 * 10),
+ (65, 1000, 10000 * 10),
+ (65, 10000, 1000 * 10)
+) AS params (numkeep, numsnaps, numiters),
+lateral snapshotbench_lifo(numkeep, numsnaps, numiters) as lifo_time_ms,
+lateral snapshotbench_fifo(numkeep, numsnaps, numiters) as fifo_time_ms;
--
2.29.2
--------------F30A5443A5D524A5CBC51AE7--
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
@ 2025-03-17 19:27 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 27+ messages in thread
From: Bruce Momjian @ 2025-03-17 19:27 UTC (permalink / raw)
To: Greg Sabino Mullane <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
On Wed, Mar 5, 2025 at 03:40:52PM -0500, Greg Sabino Mullane wrote:
> On Wed, Mar 5, 2025 at 2:43 PM Nathan Bossart <[email protected]> wrote:
>
> One other design point I wanted to bring up is whether we should bother
> generating a rollback script for the new "swap" mode. In short, I'm
> wondering if it would be unreasonable to say that, just for this mode, once
> pg_upgrade enters the file transfer step, reverting to the old cluster
> requires restoring a backup.
>
>
> I think that's a fair requirement. And like Robert, revert scripts make me
> nervous.
>
>
> * Anecdotally, I'm not sure I've ever actually seen pg_upgrade fail
> during or after file transfer, and I'm hoping to get some real data about
> that in the near future. Has anyone else dealt with such a failure?
>
>
> I've seen various failures, but they always get caught quite early. Certainly
> early enough to easily abort, fix perms/mounts/etc., then retry. I think your
> instinct is correct that this reversion is more trouble than its worth. I don't
> think the pg_upgrade docs mention taking a backup, but that's always step 0 in
> my playbook, and that's the rollback plan in the unlikely event of failure.
I avoided many optimizations in pg_upgrade in the fear they would lead
to hard-to-detect bugs, or breakage from major release changes.
pg_upgrade is probably old enough now (15 years) that we can risk these
optimizations.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Do not let urgent matters crowd out time for investment in the future.
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
@ 2025-03-17 19:34 ` Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
1 sibling, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-17 19:34 UTC (permalink / raw)
To: Greg Sabino Mullane <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers; [email protected]
On Wed, Mar 05, 2025 at 08:34:37PM -0600, Nathan Bossart wrote:
> Thank you, Greg and Robert, for sharing your thoughts. With that, here's
> what I'm considering to be a reasonably complete patch set for this
> feature. This leaves about a month for rigorous testing and editing, so
> I'm hopeful it'll be ready v18.
Here are my notes after a round of self-review.
0001:
* The documentation does not adequately describe the interaction between
--no-sync-data-files and --sync-method=syncfs.
* I really don't like the exclude_dir hack for skipping the main tablespace
directory, but I haven't thought of anything that seems better.
* I should verify that there's no path separator issues on Windows for the
exclude_dir hack. From some quick code analysis, I think it should work
fine, but I probably ought to test it out to be sure.
* The documentation needs to mention that the tablespace directories
themselves are not synchronized.
0002:
* The documentation changes are subject to update based on ongoing stats
import/export work.
* Does --statistics-only --sequence-data make any sense? It seems like it
ought to function as expected, but it's hard to see a use-case.
0003:
* Once committed, I should update one of my buildfarm animals to use
PG_TEST_PG_UPGRADE_MODE=--swap.
* For check_hard_link() and disable_old_cluster(), move the Assert() to an
"else" block with a pg_fatal() call for sturdiness.
* I need to do a thorough pass-through on all comments. Many are not
sufficiently detailed.
* The "." and ".." checks in the catalog swap code are redundant and can be
removed.
* The directory for "moved-aside" stuff should be placed within the old
cluster's corresponding tablespace directory so that no changes need to
be made to delete_old_cluster.{sh,bat}.
* Manual testing with non-default tablespaces!
Updated patches based on these notes are attached.
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-17 20:04 ` Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Robert Haas @ 2025-03-17 20:04 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Mon, Mar 17, 2025 at 3:34 PM Nathan Bossart <[email protected]> wrote:
> * Once committed, I should update one of my buildfarm animals to use
> PG_TEST_PG_UPGRADE_MODE=--swap.
It would be better if we didn't need a separate buildfarm animal to
test this, because that means you won't get notified by local testing
OR by CI if you break this. Can we instead have one test that checks
this which is part of the normal test run?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
@ 2025-03-17 20:30 ` Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-17 20:30 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Mon, Mar 17, 2025 at 04:04:45PM -0400, Robert Haas wrote:
> On Mon, Mar 17, 2025 at 3:34 PM Nathan Bossart <[email protected]> wrote:
>> * Once committed, I should update one of my buildfarm animals to use
>> PG_TEST_PG_UPGRADE_MODE=--swap.
>
> It would be better if we didn't need a separate buildfarm animal to
> test this, because that means you won't get notified by local testing
> OR by CI if you break this. Can we instead have one test that checks
> this which is part of the normal test run?
That's what I set out to do before I discovered PG_TEST_PG_UPGRADE_MODE.
The commit message for b059a24 seemed to indicate that we don't want to
automatically test all supported modes, but I agree that it would be nice
to have some basic coverage for --swap on CI/buildfarm regardless of
PG_TEST_PG_UPGRADE_MODE. How about we add a simple TAP test (attached),
and I still plan on switching a buildfarm animal to --swap for more
in-depth testing?
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-18 13:40 ` Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 15:51 ` Re: optimize file transfer in pg_upgrade Álvaro Herrera <[email protected]>
0 siblings, 2 replies; 27+ messages in thread
From: Robert Haas @ 2025-03-18 13:40 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Mon, Mar 17, 2025 at 4:30 PM Nathan Bossart <[email protected]> wrote:
> On Mon, Mar 17, 2025 at 04:04:45PM -0400, Robert Haas wrote:
> > On Mon, Mar 17, 2025 at 3:34 PM Nathan Bossart <[email protected]> wrote:
> >> * Once committed, I should update one of my buildfarm animals to use
> >> PG_TEST_PG_UPGRADE_MODE=--swap.
> >
> > It would be better if we didn't need a separate buildfarm animal to
> > test this, because that means you won't get notified by local testing
> > OR by CI if you break this. Can we instead have one test that checks
> > this which is part of the normal test run?
>
> That's what I set out to do before I discovered PG_TEST_PG_UPGRADE_MODE.
> The commit message for b059a24 seemed to indicate that we don't want to
> automatically test all supported modes, but I agree that it would be nice
> to have some basic coverage for --swap on CI/buildfarm regardless of
> PG_TEST_PG_UPGRADE_MODE. How about we add a simple TAP test (attached),
> and I still plan on switching a buildfarm animal to --swap for more
> in-depth testing?
The background here is that I'm kind of on the warpath against weird
configurations that we only test on certain buildfarm animals at the
moment, because the result of that is that CI is clean and then the
buildfarm turns red when you commit. That's an unenjoyable experience
for the committer and for everyone who looks at the buildfarm results.
The way to fix it is to stop relying on "rerun all the tests with this
weird mode flag" and rely more on tests that are designed to test that
specific flag and, ideally, that get run by in local testing or at
least by CI.
I'm not quite sure what the best thing is to do is for the pg_upgrade
tests in particular, and it may well be best to do as you propose for
now and figure that out later. But I question whether just rerunning
all of those tests with several different mode flags is the right
thing to do. Why for example does 005_char_signedness.pl need to be
checked under both --link and --clone? I would guess that there are
one or maybe two tests in src/bin/pg_upgrade/t that needs to test
--link and --clone and they should grow internal loops to do that
(when supported by the local platform) and PG_UPGRADE_TEST_MODE should
go in the garbage.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
@ 2025-03-18 14:04 ` Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
1 sibling, 1 reply; 27+ messages in thread
From: Tom Lane @ 2025-03-18 14:04 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Robert Haas <[email protected]> writes:
> I'm not quite sure what the best thing is to do is for the pg_upgrade
> tests in particular, and it may well be best to do as you propose for
> now and figure that out later. But I question whether just rerunning
> all of those tests with several different mode flags is the right
> thing to do. Why for example does 005_char_signedness.pl need to be
> checked under both --link and --clone? I would guess that there are
> one or maybe two tests in src/bin/pg_upgrade/t that needs to test
> --link and --clone and they should grow internal loops to do that
> (when supported by the local platform) and PG_UPGRADE_TEST_MODE should
> go in the garbage.
+1
I'd be particularly allergic to running 002_pg_upgrade.pl multiple
times, as that's one of our most expensive tests, and I flat out
don't believe that expending that many cycles could be justified.
Surely we can test these modes sufficiently in some much cheaper and
more targeted way.
regards, tom lane
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
@ 2025-03-18 14:12 ` Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Andres Freund @ 2025-03-18 14:12 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Robert Haas <[email protected]>; Nathan Bossart <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Hi,
On 2025-03-18 10:04:41 -0400, Tom Lane wrote:
> Robert Haas <[email protected]> writes:
> > I'm not quite sure what the best thing is to do is for the pg_upgrade
> > tests in particular, and it may well be best to do as you propose for
> > now and figure that out later. But I question whether just rerunning
> > all of those tests with several different mode flags is the right
> > thing to do. Why for example does 005_char_signedness.pl need to be
> > checked under both --link and --clone? I would guess that there are
> > one or maybe two tests in src/bin/pg_upgrade/t that needs to test
> > --link and --clone and they should grow internal loops to do that
> > (when supported by the local platform) and PG_UPGRADE_TEST_MODE should
> > go in the garbage.
>
> +1
>
> I'd be particularly allergic to running 002_pg_upgrade.pl multiple
> times, as that's one of our most expensive tests, and I flat out
> don't believe that expending that many cycles could be justified.
> Surely we can test these modes sufficiently in some much cheaper and
> more targeted way.
+1
It's useful to have coverage of as many object types as possible in pg_upgrade
- hence 002_pg_upgrade.pl. It helps us to find problems in new code that
didn't think about pg_upgrade.
But that doesn't mean that it's a good idea to run all other pg_upgrade tests
the same way, to the contrary - the cost is too high.
Even leaving runtime aside, I have a hard time believing that --link, --clone,
--swap benefit from running the same way as 002_pg_upgrade.pl - the
implementation of those flags is on a lower level and works the same across
e.g. different index AMs.
I'd go so far as to say that 002_pg_upgrade.pl style testing actually makes it
*harder* to diagnose problems related to things like --link, because there are
no targeted tests, but just a huge set of things that maybe allow to infer
some bug if you spend a lot of time.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
@ 2025-03-18 17:29 ` Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-18 17:29 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Tue, Mar 18, 2025 at 10:12:51AM -0400, Andres Freund wrote:
> On 2025-03-18 10:04:41 -0400, Tom Lane wrote:
>> Robert Haas <[email protected]> writes:
>> > I'm not quite sure what the best thing is to do is for the pg_upgrade
>> > tests in particular, and it may well be best to do as you propose for
>> > now and figure that out later. But I question whether just rerunning
>> > all of those tests with several different mode flags is the right
>> > thing to do. Why for example does 005_char_signedness.pl need to be
>> > checked under both --link and --clone? I would guess that there are
>> > one or maybe two tests in src/bin/pg_upgrade/t that needs to test
>> > --link and --clone and they should grow internal loops to do that
>> > (when supported by the local platform) and PG_UPGRADE_TEST_MODE should
>> > go in the garbage.
>>
>> +1
>>
>> I'd be particularly allergic to running 002_pg_upgrade.pl multiple
>> times, as that's one of our most expensive tests, and I flat out
>> don't believe that expending that many cycles could be justified.
>> Surely we can test these modes sufficiently in some much cheaper and
>> more targeted way.
>
> +1
Here is a first sketch at a test that cycles through all the transfer modes
and makes sure they succeed or fail with an error along the lines of "not
supported on this platform." Each test verifies that some very simple
objects make it to the new version, which we could of course expand on.
Would something like this suffice?
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-18 17:37 ` Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Andres Freund @ 2025-03-18 17:37 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Hi,
On 2025-03-18 12:29:02 -0500, Nathan Bossart wrote:
> Here is a first sketch at a test that cycles through all the transfer modes
> and makes sure they succeed or fail with an error along the lines of "not
> supported on this platform." Each test verifies that some very simple
> objects make it to the new version, which we could of course expand on.
> Would something like this suffice?
I'd add a few more complications:
- Create and test a relation that was rewritten, to ensure we test the
relfilenode != oid case and one that isn't rewritten.
- Perhaps create a tablespace?
- Do we need a new old cluster for each of the modes? That seems like wasted
time? I guess it's required for --link...
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
@ 2025-03-18 17:47 ` Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-18 17:47 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Tue, Mar 18, 2025 at 01:37:02PM -0400, Andres Freund wrote:
> I'd add a few more complications:
>
> - Create and test a relation that was rewritten, to ensure we test the
> relfilenode != oid case and one that isn't rewritten.
+1
> - Perhaps create a tablespace?
+1, I don't think we have much, if any, coverage of pg_upgrade with
non-default tablespaces.
> - Do we need a new old cluster for each of the modes? That seems like wasted
> time? I guess it's required for --link...
It'll also be needed for --swap. We could optionally save the old cluster
for a couple of modes if we really wanted to. *shrug*
I'll work on the first two...
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-18 17:50 ` Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Andres Freund @ 2025-03-18 17:50 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On 2025-03-18 12:47:01 -0500, Nathan Bossart wrote:
> On Tue, Mar 18, 2025 at 01:37:02PM -0400, Andres Freund wrote:
> > - Do we need a new old cluster for each of the modes? That seems like wasted
> > time? I guess it's required for --link...
>
> It'll also be needed for --swap. We could optionally save the old cluster
> for a couple of modes if we really wanted to. *shrug*
Don't worry about it, I think the template initdb stuff should make it cheap enough...
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
@ 2025-03-18 19:08 ` Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-18 19:08 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Tue, Mar 18, 2025 at 01:50:10PM -0400, Andres Freund wrote:
> On 2025-03-18 12:47:01 -0500, Nathan Bossart wrote:
>> On Tue, Mar 18, 2025 at 01:37:02PM -0400, Andres Freund wrote:
>> > - Do we need a new old cluster for each of the modes? That seems like wasted
>> > time? I guess it's required for --link...
>>
>> It'll also be needed for --swap. We could optionally save the old cluster
>> for a couple of modes if we really wanted to. *shrug*
>
> Don't worry about it, I think the template initdb stuff should make it cheap enough...
Cool. I realize now why there's poor coverage for pg_upgrade with
tablespaces: you can't upgrade between the same version with tablespaces
(presumably due to the version-specific subdirectory conflict). I don't
know if the regression tests leave around any tablespaces for the
cross-version pg_upgrade tests, but that's probably the best we can do at
the moment.
For now, here's a new version of the test with a rewritten table. I also
tried to fix the expected error regex to handle some of the other error
messages for unsupported modes (as revealed by cfbot).
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-19 02:14 ` Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:41 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
0 siblings, 2 replies; 27+ messages in thread
From: Nathan Bossart @ 2025-03-19 02:14 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Tue, Mar 18, 2025 at 02:08:42PM -0500, Nathan Bossart wrote:
> For now, here's a new version of the test with a rewritten table. I also
> tried to fix the expected error regex to handle some of the other error
> messages for unsupported modes (as revealed by cfbot).
And here is a new version of the full patch set.
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-19 15:31 ` Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
1 sibling, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-19 15:31 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Tue, Mar 18, 2025 at 09:14:22PM -0500, Nathan Bossart wrote:
> And here is a new version of the full patch set.
I'm currently planning to commit this sometime early-ish next week. One
notable loose end is the lack of a pg_upgrade test with a non-default
tablespace, but that is an existing problem that IMHO is best handled
separately (since we can only test it in cross-version upgrades).
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-19 16:28 ` Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Tom Lane @ 2025-03-19 16:28 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Andres Freund <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Nathan Bossart <[email protected]> writes:
> I'm currently planning to commit this sometime early-ish next week. One
> notable loose end is the lack of a pg_upgrade test with a non-default
> tablespace, but that is an existing problem that IMHO is best handled
> separately (since we can only test it in cross-version upgrades).
Agreed that that shouldn't block this, but we need some kind of
plan for testing it better.
regards, tom lane
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
@ 2025-03-19 16:44 ` Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Andres Freund @ 2025-03-19 16:44 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Hi,
On 2025-03-19 12:28:33 -0400, Tom Lane wrote:
> Nathan Bossart <[email protected]> writes:
> > I'm currently planning to commit this sometime early-ish next week. One
> > notable loose end is the lack of a pg_upgrade test with a non-default
> > tablespace, but that is an existing problem that IMHO is best handled
> > separately (since we can only test it in cross-version upgrades).
>
> Agreed that that shouldn't block this, but we need some kind of
> plan for testing it better.
Yea, this is really suboptimal.
Shouldn't allow_in_place_tablespaces be sufficient to deal with that scenario?
Or at least it should make it reasonably easy to cope if it doesn't already
suffice?
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
@ 2025-03-19 19:32 ` Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-19 19:32 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Wed, Mar 19, 2025 at 12:44:38PM -0400, Andres Freund wrote:
> On 2025-03-19 12:28:33 -0400, Tom Lane wrote:
>> Nathan Bossart <[email protected]> writes:
>> > I'm currently planning to commit this sometime early-ish next week. One
>> > notable loose end is the lack of a pg_upgrade test with a non-default
>> > tablespace, but that is an existing problem that IMHO is best handled
>> > separately (since we can only test it in cross-version upgrades).
>>
>> Agreed that that shouldn't block this, but we need some kind of
>> plan for testing it better.
>
> Yea, this is really suboptimal.
>
> Shouldn't allow_in_place_tablespaces be sufficient to deal with that scenario?
> Or at least it should make it reasonably easy to cope if it doesn't already
> suffice?
Unfortunately, pg_upgrade can't yet handle in-place tablespaces. One
reason is that pg_tablespace_location() returns a relative path for those
(e.g., "pg_tblspc/123456"). We'd also need to adjust init_tablespaces() to
not fail if all the tablespaces are in-place. There may be other reasons,
too. I'm confident we could get it working, but I'm not too excited about
trying to sneak this into v18.
In addition to testing with in-place tablespaces, we might also want to
teach the transfer modes test to do cross-version testing when possible.
In that case, we can test normal (non-in-place) tablespaces. However, that
would be limited to the buildfarm.
Does this seem like a reasonable plan for v19?
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-19 21:28 ` Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-19 21:28 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Wed, Mar 19, 2025 at 02:32:01PM -0500, Nathan Bossart wrote:
> In addition to testing with in-place tablespaces, we might also want to
> teach the transfer modes test to do cross-version testing when possible.
> In that case, we can test normal (non-in-place) tablespaces. However, that
> would be limited to the buildfarm.
Actually, this one was pretty easy to do.
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-20 02:02 ` Nathan Bossart <[email protected]>
2025-03-20 16:11 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-04-28 18:26 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 2 replies; 27+ messages in thread
From: Nathan Bossart @ 2025-03-20 02:02 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Wed, Mar 19, 2025 at 04:28:23PM -0500, Nathan Bossart wrote:
> On Wed, Mar 19, 2025 at 02:32:01PM -0500, Nathan Bossart wrote:
>> In addition to testing with in-place tablespaces, we might also want to
>> teach the transfer modes test to do cross-version testing when possible.
>> In that case, we can test normal (non-in-place) tablespaces. However, that
>> would be limited to the buildfarm.
>
> Actually, this one was pretty easy to do.
And here is yet another new version of the full patch set. I'm planning to
commit 0001 (the new pg_upgrade transfer mode test) tomorrow so that I can
deal with any buildfarm indigestion before committing swap mode. I did run
the test locally for upgrades from v9.6, v13, and v17, but who knows what
unique configurations I've failed to anticipate...
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-20 16:11 ` Nathan Bossart <[email protected]>
2025-03-20 20:23 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
1 sibling, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-20 16:11 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Wed, Mar 19, 2025 at 09:02:42PM -0500, Nathan Bossart wrote:
> On Wed, Mar 19, 2025 at 04:28:23PM -0500, Nathan Bossart wrote:
>> On Wed, Mar 19, 2025 at 02:32:01PM -0500, Nathan Bossart wrote:
>>> In addition to testing with in-place tablespaces, we might also want to
>>> teach the transfer modes test to do cross-version testing when possible.
>>> In that case, we can test normal (non-in-place) tablespaces. However, that
>>> would be limited to the buildfarm.
>>
>> Actually, this one was pretty easy to do.
>
> And here is yet another new version of the full patch set. I'm planning to
> commit 0001 (the new pg_upgrade transfer mode test) tomorrow so that I can
> deal with any buildfarm indigestion before committing swap mode. I did run
> the test locally for upgrades from v9.6, v13, and v17, but who knows what
> unique configurations I've failed to anticipate...
As promised, I've committed just 0001 for now. I'll watch closely for any
issues in the buildfarm.
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 16:11 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-20 20:23 ` Nathan Bossart <[email protected]>
2025-03-25 21:03 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
0 siblings, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-03-20 20:23 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Thu, Mar 20, 2025 at 11:11:46AM -0500, Nathan Bossart wrote:
> As promised, I've committed just 0001 for now. I'll watch closely for any
> issues in the buildfarm.
Seeing none, here's is a rebased patch set without 0001. The only changes
are some fleshed-out comments and commit messages. I'm still aiming to
commit this sometime early next week.
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 16:11 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 20:23 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-25 21:03 ` Nathan Bossart <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Nathan Bossart @ 2025-03-25 21:03 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Thu, Mar 20, 2025 at 03:23:13PM -0500, Nathan Bossart wrote:
> I'm still aiming to commit this sometime early next week.
Committed. Thanks to everyone who chimed in on this thread.
While writing the attributions, I noticed that nobody seems to have
commented specifically on 0001. The closest thing to a review I see is
Greg's note upthread [0]. This patch is a little bigger than what I'd
ordinarily feel comfortable with committing unilaterally, but it's been
posted in its current form since February 28th, this thread has gotten a
decent amount of traffic since then, and it's not a huge change ("9 files
changed, 96 insertions(+), 37 deletions(-)"). I'm happy to address any
post-commit feedback that folks have. As noted earlier [1], I'm not wild
about how it's implemented, but this is the nicest approach I've thought of
thus far.
I also wanted to draw attention to this note in 0003:
/*
* XXX: The below line is a hack to deal with the fact that we
* presently don't have an easy way to find the corresponding new
* tablespace's path. This will need to be fixed if/when we add
* pg_upgrade support for in-place tablespaces.
*/
new_tablespace = old_tablespace;
I intend to address this in v19, primarily to enable same-version
pg_upgrade testing with non-default tablespaces. My current thinking is
that we should have pg_upgrade also gather the new cluster tablespace
information and map them to the corresponding tablespaces on the old
cluster. This might require some refactoring in pg_upgrade. In any case,
I didn't feel this should block the feature for v18.
[0] https://postgr.es/m/CAKAnmm%2Bi3Q1pZ05N_b8%3DS3B%3DrztQDn--HoW8BRKVtCg53r8NiQ%40mail.gmail.com
[1] https://postgr.es/m/Z9h5Spp76EBygyEL%40nathan
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-04-28 18:26 ` Nathan Bossart <[email protected]>
2025-12-28 07:00 ` Re: optimize file transfer in pg_upgrade Alexander Lakhin <[email protected]>
1 sibling, 1 reply; 27+ messages in thread
From: Nathan Bossart @ 2025-04-28 18:26 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Andres Freund <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On Mon, Apr 28, 2025 at 09:00:01PM +0300, Alexander Lakhin wrote:
> Thank you for the references! Unfortunately I still can't see where the
> lack of upgrade log files is discussed.
That was briefly discussed here:
https://postgr.es/m/644cf995-e3a5-4f69-9398-7db500e2673d%40dunslane.net
One other potential problem with this test is that we reuse the directory
names for each transfer mode. That seems easy enough to fix.
--
nathan
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-04-28 18:26 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-12-28 07:00 ` Alexander Lakhin <[email protected]>
0 siblings, 0 replies; 27+ messages in thread
From: Alexander Lakhin @ 2025-12-28 07:00 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Andres Freund <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Hello Nathan,
28.04.2025 21:26, Nathan Bossart wrote:
> One other potential problem with this test is that we reuse the directory
> names for each transfer mode. That seems easy enough to fix.
FWIW, I've counted seven 006_transfer_modes failures happened during this
year. Five are from Windows animals:
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=drongo&dt=2025-04-08%2004%3A18%3A15
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=fairywren&dt=2025-04-21%2008%3A03%3A06
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=fairywren&dt=2025-07-21%2012%3A35%3A58
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=fairywren&dt=2025-08-22%2000%3A04%3A05
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=drongo&dt=2025-12-28%2003%3A43%3A24
And two from culicidae, which tests EXEC_BACKEND and thus suffers from [1]
([2]):
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=culicidae&dt=2025-11-22%2012%3A31%3A23
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=culicidae&dt=2025-12-14%2018%3A24%3A48
Probably, this number could justify improving the test so that we can
identify the failure reason for sure looking at the upgrade logs. As of
now, we can only guess it, based on the animals' specifics...
[1]
https://wiki.postgresql.org/wiki/Known_Buildfarm_Test_Failures#culicidae_failed_to_restart_server_du...
[2] https://www.postgresql.org/message-id/[email protected]
Best regards,
Alexander
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
@ 2025-03-19 15:41 ` Andres Freund <[email protected]>
1 sibling, 0 replies; 27+ messages in thread
From: Andres Freund @ 2025-03-19 15:41 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
Hi,
On 2025-03-18 21:14:22 -0500, Nathan Bossart wrote:
> From 8b6a5e0148c2f7a663f5003f12ae9461d2b06a5c Mon Sep 17 00:00:00 2001
> From: Nathan Bossart <[email protected]>
> Date: Tue, 18 Mar 2025 20:58:07 -0500
> Subject: [PATCH v7 1/4] Add test for pg_upgrade file transfer modes.
>
> This new test checks all of pg_upgrade's file transfer modes. For
> each mode, we verify that pg_upgrade either succeeds (and some test
> objects successfully reach the new version) or fails with an error
> that indicates the mode is not supported on the current platform.
LGTM. I'm sure we could do more than the test does today, but I think it's a
good improvement.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: optimize file transfer in pg_upgrade
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
@ 2025-03-18 15:51 ` Álvaro Herrera <[email protected]>
1 sibling, 0 replies; 27+ messages in thread
From: Álvaro Herrera @ 2025-03-18 15:51 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Greg Sabino Mullane <[email protected]>; pgsql-hackers; [email protected]
On 2025-Mar-18, Robert Haas wrote:
> The background here is that I'm kind of on the warpath against weird
> configurations that we only test on certain buildfarm animals at the
> moment, because the result of that is that CI is clean and then the
> buildfarm turns red when you commit. That's an unenjoyable experience
> for the committer and for everyone who looks at the buildfarm results.
> The way to fix it is to stop relying on "rerun all the tests with this
> weird mode flag" and rely more on tests that are designed to test that
> specific flag and, ideally, that get run by in local testing or at
> least by CI.
FWIW this is exactly the rationale that got me writing an email on the
Ashutosh's thread for a new pg_dump/restore test under
002_pg_upgrade.pl, whereby I was saying that we should not hide it
behind PG_TEST_EXTRA which almost nobody would remember to use. But I
discarded that draft, because that had actually been Ashutosh's idea at
some point in the thread and had been discarded because of the runtime
increase it'd cause. But, somehow, I still don't believe the theory
that it's such a bad idea to add a few seconds so that we have such a
comprehensive pg_dump test, with much less programmer overhead than
pg_dump's own weird enormous test script.
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
"Cada quien es cada cual y baja las escaleras como quiere" (JMSerrat)
^ permalink raw reply [nested|flat] 27+ messages in thread
end of thread, other threads:[~2025-12-28 07:00 UTC | newest]
Thread overview: 27+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-11-11 22:40 [PATCH v2 1/1] resownerbench Heikki Linnakangas <[email protected]>
2025-03-17 19:27 ` Re: optimize file transfer in pg_upgrade Bruce Momjian <[email protected]>
2025-03-17 19:34 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-17 20:04 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-17 20:30 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 13:40 ` Re: optimize file transfer in pg_upgrade Robert Haas <[email protected]>
2025-03-18 14:04 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-18 14:12 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:29 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:37 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 17:47 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-18 17:50 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 19:08 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 02:14 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 15:31 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 16:28 ` Re: optimize file transfer in pg_upgrade Tom Lane <[email protected]>
2025-03-19 16:44 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-19 19:32 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-19 21:28 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 02:02 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 16:11 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-20 20:23 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-03-25 21:03 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-04-28 18:26 ` Re: optimize file transfer in pg_upgrade Nathan Bossart <[email protected]>
2025-12-28 07:00 ` Re: optimize file transfer in pg_upgrade Alexander Lakhin <[email protected]>
2025-03-19 15:41 ` Re: optimize file transfer in pg_upgrade Andres Freund <[email protected]>
2025-03-18 15:51 ` Re: optimize file transfer in pg_upgrade Álvaro Herrera <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox