From: Nathan Bossart <nathandbossart@gmail.com>
To: Nitin Motiani <nitinmotiani@google.com>
Cc: Hannu Krosing <hannuk@google.com>
Cc: Tom Lane <tgl@sss.pgh.pa.us>
Cc: PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>
Subject: Re: Horribly slow pg_upgrade performance with many Large Objects
Date: Thu, 10 Jul 2025 16:42:23 -0500
Message-ID: <aHAzv9pgJh2Wa3M1@nathan> (raw)
In-Reply-To: <CAH5HC942o6j1qyVysiBtO+Vpb=XOBPbf3rKqsOLtpDpp0SzjkA@mail.gmail.com>
References: <Z_VTucTiS01p-d6z@nathan>
<4050387.1744132029@sss.pgh.pa.us>
<aBkQLSkx1zUJ-LwJ@nathan>
<aBtzZ4PyccUVCcyA@nathan>
<CAMT0RQTXiqH7zdQEVSVd2L7_Cw4wQ1eHOD8hfZ+0vecMXJWc-w@mail.gmail.com>
<aG2ISJC__3lQwRqY@nathan>
<CAMT0RQStPtHfKwowd88Q0tynX0x=uJSKn=ihP8syhDJ6cH3DHQ@mail.gmail.com>
<aG6srPib60NzJBeg@nathan>
<CAMT0RQRT3KkwoQkx=YBdd_o9SqXxOt16fqQsscH_n+40Z57N7w@mail.gmail.com>
<CAH5HC942o6j1qyVysiBtO+Vpb=XOBPbf3rKqsOLtpDpp0SzjkA@mail.gmail.com>
On Thu, Jul 10, 2025 at 06:05:26PM +0530, Nitin Motiani wrote:
> - * pg_largeobject_metadata, after the dump is restored.
> + * pg_largeobject_metadata, after the dump is restored. In versions
> + * before v12, this is done via proper large object commands. In
> + * newer versions, we dump the content of pg_largeobject_metadata and
> + * any associated pg_shdepend rows, which is faster to restore.
> */
>
> Should the comment provide further detail on why this is only being done
> for v12 and above?
Yes. I've fixed this in v3.
--
nathan
Attachments:v3-0001-pg_upgrade-Use-COPY-for-large-object-metadata.patch (text/plain, 8.4K) download
From b784cd881933a1c01f4ea143f06bf77972924459 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Thu, 10 Jul 2025 16:20:33 -0500
Subject: [PATCH v3 1/1] pg_upgrade: Use COPY for large object metadata.
Reported-by: Hannu Krosing <hannuk@google.com>
Suggested-by: Tom Lane <tgl@sss.pgh.pa.us>
Reviewed-by: Hannu Krosing <hannuk@google.com>
Reviewed-by: Nitin Motiani <nitinmotiani@google.com>
Discussion: https://postgr.es/m/CAMT0RQSS-6qLH%2BzYsOeUbAYhop3wmQTkNmQpo5--QRDUR%2BqYmQ%40mail.gmail.com
---
src/bin/pg_dump/pg_backup_archiver.c | 15 +++++
src/bin/pg_dump/pg_dump.c | 84 ++++++++++++++++++++++++++--
src/bin/pg_dump/t/002_pg_dump.pl | 4 +-
3 files changed, 97 insertions(+), 6 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.cindex 197c1295d93..1b1c61e5ce5 100644--- a/src/bin/pg_dump/pg_backup_archiver.c+++ b/src/bin/pg_dump/pg_backup_archiver.c@@ -31,6 +31,8 @@
#endif
#include "catalog/pg_class_d.h"
+#include "catalog/pg_largeobject_metadata.h"+#include "catalog/pg_shdepend.h"
#include "common/string.h"
#include "compress_io.h"
#include "dumputils.h"
@@ -2974,6 +2976,19 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
int res = REQ_SCHEMA | REQ_DATA;
RestoreOptions *ropt = AH->public.ropt;
+ /*+ * For binary upgrade mode, dump pg_largeobject_metadata and the+ * associated pg_shdepend rows. This is faster to restore than the+ * equivalent set of large object commands. We can only do this for+ * upgrades from v12 and newer; in older versions, pg_largeobject_metadata+ * was created WITH OIDS, so the OID column is hidden and won't be dumped.+ */+ if (ropt->binary_upgrade && AH->public.remoteVersion >= 120000 &&+ strcmp(te->desc, "TABLE DATA") == 0 &&+ (te->catalogId.oid == LargeObjectMetadataRelationId ||+ te->catalogId.oid == SharedDependRelationId))+ return REQ_DATA;+
/* These items are treated specially */
if (strcmp(te->desc, "ENCODING") == 0 ||
strcmp(te->desc, "STDSTRINGS") == 0 ||
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.cindex 1937997ea67..bdea0ef7c50 100644--- a/src/bin/pg_dump/pg_dump.c+++ b/src/bin/pg_dump/pg_dump.c@@ -49,8 +49,10 @@
#include "catalog/pg_class_d.h"
#include "catalog/pg_default_acl_d.h"
#include "catalog/pg_largeobject_d.h"
+#include "catalog/pg_largeobject_metadata_d.h"
#include "catalog/pg_proc_d.h"
#include "catalog/pg_publication_d.h"
+#include "catalog/pg_shdepend.h"
#include "catalog/pg_subscription_d.h"
#include "catalog/pg_type_d.h"
#include "common/connect.h"
@@ -209,6 +211,12 @@ static int nbinaryUpgradeClassOids = 0;
static SequenceItem *sequences = NULL;
static int nsequences = 0;
+/*+ * For binary upgrade, the dump ID of pg_largeobject_metadata is saved for use+ * as a dependency for pg_shdepend and any large object comments/seclabels.+ */+static DumpId lo_metadata_dumpId;+
/* Maximum number of relations to fetch in a fetchAttributeStats() call. */
#define MAX_ATTR_STATS_RELS 64
@@ -1085,6 +1093,36 @@ main(int argc, char **argv)
if (!dopt.dumpData && dopt.sequence_data)
getTableData(&dopt, tblinfo, numTables, RELKIND_SEQUENCE);
+ /*+ * For binary upgrade mode, dump pg_largeobject_metadata and the+ * associated pg_shdepend rows. This is faster to restore than the+ * equivalent set of large object commands. We can only do this for+ * upgrades from v12 and newer; in older versions, pg_largeobject_metadata+ * was created WITH OIDS, so the OID column is hidden and won't be dumped.+ */+ if (dopt.binary_upgrade && fout->remoteVersion >= 120000)+ {+ TableInfo *lo_metadata = findTableByOid(LargeObjectMetadataRelationId);+ TableInfo *shdepend = findTableByOid(SharedDependRelationId);++ makeTableDataInfo(&dopt, lo_metadata);+ makeTableDataInfo(&dopt, shdepend);++ /*+ * Save pg_largeobject_metadata's dump ID for use as a dependency on+ * pg_shdepend and any large object comments/seclabels.+ */+ lo_metadata_dumpId = lo_metadata->dataObj->dobj.dumpId;+ addObjectDependency(&shdepend->dataObj->dobj, lo_metadata_dumpId);++ /*+ * Only dump large object shdepend rows for this database.+ */+ shdepend->dataObj->filtercond = "WHERE classid = 'pg_largeobject'::regclass "+ "AND dbid = (SELECT oid FROM pg_database "+ " WHERE datname = current_database())";+ }+
/*
* In binary-upgrade mode, we do not have to worry about the actual LO
* data or the associated metadata that resides in the pg_largeobject and
@@ -3924,10 +3962,31 @@ getLOs(Archive *fout)
* as it will be copied by pg_upgrade, which simply copies the
* pg_largeobject table. We *do* however dump out anything but the
* data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.+ * pg_largeobject_metadata, after the dump is restored. In versions+ * before v12, this is done via proper large object commands. In+ * newer versions, we dump the content of pg_largeobject_metadata and+ * any associated pg_shdepend rows, which is faster to restore. (On+ * <v12, pg_largeobject_metadata was created WITH OIDS, so the OID+ * column is hidden and won't be dumped.)
*/
if (dopt->binary_upgrade)
- loinfo->dobj.dump &= ~DUMP_COMPONENT_DATA;+ {+ if (fout->remoteVersion >= 120000)+ {+ loinfo->dobj.dump &= ~(DUMP_COMPONENT_DATA | DUMP_COMPONENT_ACL | DUMP_COMPONENT_DEFINITION);++ /*+ * Mark the large object as dependent on+ * pg_largeobject_metadata so that any large object+ * comments/seclables are dumped after it.+ */+ loinfo->dobj.dependencies = (DumpId *) pg_malloc(sizeof(DumpId));+ loinfo->dobj.dependencies[0] = lo_metadata_dumpId;+ loinfo->dobj.nDeps = loinfo->dobj.allocDeps = 1;+ }+ else+ loinfo->dobj.dump &= ~DUMP_COMPONENT_DATA;+ }
/*
* Create a "BLOBS" data item for the group, too. This is just a
@@ -9039,8 +9098,20 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
if (tbinfo->relkind == RELKIND_SEQUENCE)
continue;
- /* Don't bother with uninteresting tables, either */- if (!tbinfo->interesting)+ /*+ * Don't bother with uninteresting tables, either. For binary+ * upgrades, this is bypassed for pg_largeobject_metadata and+ * pg_shdepend so that the columns names are collected for the+ * corresponding COPY commands. Restoring the data for those catalogs+ * is faster than restoring the equivalent set of large object+ * commands. We can only do this for upgrades from v12 and newer; in+ * older versions, pg_largeobject_metadata was created WITH OIDS, so+ * the OID column is hidden and won't be dumped.+ */+ if (!tbinfo->interesting &&+ !(fout->dopt->binary_upgrade && fout->remoteVersion >= 120000 &&+ (tbinfo->dobj.catId.oid == LargeObjectMetadataRelationId ||+ tbinfo->dobj.catId.oid == SharedDependRelationId)))
continue;
/* OK, we need info for this table */
@@ -9244,7 +9315,10 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
pg_fatal("unrecognized table OID %u", attrelid);
/* cross-check that we only got requested tables */
if (tbinfo->relkind == RELKIND_SEQUENCE ||
- !tbinfo->interesting)+ (!tbinfo->interesting &&+ !(fout->dopt->binary_upgrade && fout->remoteVersion >= 120000 &&+ (tbinfo->dobj.catId.oid == LargeObjectMetadataRelationId ||+ tbinfo->dobj.catId.oid == SharedDependRelationId))))
pg_fatal("unexpected column data for table \"%s\"",
tbinfo->dobj.name);
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.plindex 2485d8f360e..d8330e2bd17 100644--- a/src/bin/pg_dump/t/002_pg_dump.pl+++ b/src/bin/pg_dump/t/002_pg_dump.pl@@ -1087,6 +1087,7 @@ my %tests = (
test_schema_plus_large_objects => 1,
},
unlike => {
+ binary_upgrade => 1,
no_large_objects => 1,
no_owner => 1,
schema_only => 1,
@@ -1605,6 +1606,7 @@ my %tests = (
test_schema_plus_large_objects => 1,
},
unlike => {
+ binary_upgrade => 1,
schema_only => 1,
schema_only_with_statistics => 1,
no_large_objects => 1,
@@ -4612,9 +4614,9 @@ my %tests = (
no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
- binary_upgrade => 1,
},
unlike => {
+ binary_upgrade => 1,
no_large_objects => 1,
no_privs => 1,
schema_only => 1,
--
2.39.5 (Apple Git-154)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: nathandbossart@gmail.com, nitinmotiani@google.com, hannuk@google.com, tgl@sss.pgh.pa.us, pgsql-hackers@lists.postgresql.org
Subject: Re: Horribly slow pg_upgrade performance with many Large Objects
In-Reply-To: <aHAzv9pgJh2Wa3M1@nathan>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox