public inbox for [email protected]
help / color / mirror / Atom feed[PATCH] pg_dump: use a cursor in getBlobs..
26+ messages / 3 participants
[nested] [flat]
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* [PATCH] pg_dump: use a cursor in getBlobs..
@ 2021-03-09 20:06 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Justin Pryzby @ 2021-03-09 20:06 UTC (permalink / raw)
..to mitigate huge memory use in the case of millions of large objects
---
src/bin/pg_dump/pg_dump.c | 96 +++++++++++++++++++++------------------
1 file changed, 52 insertions(+), 44 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa02ada079..3fd7f48605 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3333,7 +3333,7 @@ getBlobs(Archive *fout)
BlobInfo *binfo;
DumpableObject *bdata;
PGresult *res;
- int ntups;
+ int ntups, total = 0;
int i;
int i_oid;
int i_lomowner;
@@ -3341,9 +3341,12 @@ getBlobs(Archive *fout)
int i_rlomacl;
int i_initlomacl;
int i_initrlomacl;
+ const char *blobFetchQry = "FETCH 1000 IN blob";
pg_log_info("reading large objects");
+ appendPQExpBuffer(blobQry, "DECLARE blob CURSOR FOR ");
+
/* Fetch BLOB OIDs, and owner/ACL data if >= 9.0 */
if (fout->remoteVersion >= 90600)
{
@@ -3393,58 +3396,66 @@ getBlobs(Archive *fout)
"NULL::oid AS initrlomacl "
" FROM pg_largeobject");
- res = ExecuteSqlQuery(fout, blobQry->data, PGRES_TUPLES_OK);
+ ExecuteSqlStatement(fout, blobQry->data);
+ destroyPQExpBuffer(blobQry);
- i_oid = PQfnumber(res, "oid");
- i_lomowner = PQfnumber(res, "rolname");
- i_lomacl = PQfnumber(res, "lomacl");
- i_rlomacl = PQfnumber(res, "rlomacl");
- i_initlomacl = PQfnumber(res, "initlomacl");
- i_initrlomacl = PQfnumber(res, "initrlomacl");
+ do {
+ res = ExecuteSqlQuery(fout, blobFetchQry, PGRES_TUPLES_OK);
- ntups = PQntuples(res);
+ i_oid = PQfnumber(res, "oid");
+ i_lomowner = PQfnumber(res, "rolname");
+ i_lomacl = PQfnumber(res, "lomacl");
+ i_rlomacl = PQfnumber(res, "rlomacl");
+ i_initlomacl = PQfnumber(res, "initlomacl");
+ i_initrlomacl = PQfnumber(res, "initrlomacl");
- /*
- * Each large object has its own BLOB archive entry.
- */
- binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
+ ntups = PQntuples(res);
+ total += ntups;
- for (i = 0; i < ntups; i++)
- {
- binfo[i].dobj.objType = DO_BLOB;
- binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
- binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
- AssignDumpId(&binfo[i].dobj);
+ /*
+ * Each large object has its own BLOB archive entry.
+ */
+ binfo = (BlobInfo *) pg_malloc(ntups * sizeof(BlobInfo));
- binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
- binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
- binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
- binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
- binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
- binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+ for (i = 0; i < ntups; i++)
+ {
+ binfo[i].dobj.objType = DO_BLOB;
+ binfo[i].dobj.catId.tableoid = LargeObjectRelationId;
+ binfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_oid));
+ AssignDumpId(&binfo[i].dobj);
+
+ binfo[i].dobj.name = pg_strdup(PQgetvalue(res, i, i_oid));
+ binfo[i].rolname = pg_strdup(PQgetvalue(res, i, i_lomowner));
+ binfo[i].blobacl = pg_strdup(PQgetvalue(res, i, i_lomacl));
+ binfo[i].rblobacl = pg_strdup(PQgetvalue(res, i, i_rlomacl));
+ binfo[i].initblobacl = pg_strdup(PQgetvalue(res, i, i_initlomacl));
+ binfo[i].initrblobacl = pg_strdup(PQgetvalue(res, i, i_initrlomacl));
+
+ if (PQgetisnull(res, i, i_lomacl) &&
+ PQgetisnull(res, i, i_rlomacl) &&
+ PQgetisnull(res, i, i_initlomacl) &&
+ PQgetisnull(res, i, i_initrlomacl))
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
- if (PQgetisnull(res, i, i_lomacl) &&
- PQgetisnull(res, i, i_rlomacl) &&
- PQgetisnull(res, i, i_initlomacl) &&
- PQgetisnull(res, i, i_initrlomacl))
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_ACL;
+ /*
+ * In binary-upgrade mode for blobs, we do *not* dump out the blob
+ * data, as it will be copied by pg_upgrade, which simply copies the
+ * pg_largeobject table. We *do* however dump out anything but the
+ * data, as pg_upgrade copies just pg_largeobject, but not
+ * pg_largeobject_metadata, after the dump is restored.
+ */
+ if (dopt->binary_upgrade)
+ binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
+ }
- /*
- * In binary-upgrade mode for blobs, we do *not* dump out the blob
- * data, as it will be copied by pg_upgrade, which simply copies the
- * pg_largeobject table. We *do* however dump out anything but the
- * data, as pg_upgrade copies just pg_largeobject, but not
- * pg_largeobject_metadata, after the dump is restored.
- */
- if (dopt->binary_upgrade)
- binfo[i].dobj.dump &= ~DUMP_COMPONENT_DATA;
- }
+ PQclear(res);
+ } while (ntups != 0);
/*
* If we have any large objects, a "BLOBS" archive entry is needed. This
* is just a placeholder for sorting; it carries no data now.
*/
- if (ntups > 0)
+ if (total > 0)
{
bdata = (DumpableObject *) pg_malloc(sizeof(DumpableObject));
bdata->objType = DO_BLOB_DATA;
@@ -3452,9 +3463,6 @@ getBlobs(Archive *fout)
AssignDumpId(bdata);
bdata->name = pg_strdup("BLOBS");
}
-
- PQclear(res);
- destroyPQExpBuffer(blobQry);
}
/*
--
2.17.0
--gatW/ieO32f1wygP--
^ permalink raw reply [nested|flat] 26+ messages in thread
* Re: Adding facility for injection points (or probe points?) for more advanced tests
@ 2023-11-27 22:36 Michael Paquier <[email protected]>
0 siblings, 1 reply; 26+ messages in thread
From: Michael Paquier @ 2023-11-27 22:36 UTC (permalink / raw)
To: Ashutosh Bapat <[email protected]>; +Cc: Postgres hackers <[email protected]>
On Mon, Nov 27, 2023 at 12:14:05PM +0530, Ashutosh Bapat wrote:
> Since you wroten "(still need to improve ...) I thought you are
> working on v6. No problem. Sorry for the confusion.
I see why my previous message could be confusing. Sorry about that.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 26+ messages in thread
* Re: Adding facility for injection points (or probe points?) for more advanced tests
@ 2023-12-11 05:39 Dilip Kumar <[email protected]>
parent: Michael Paquier <[email protected]>
0 siblings, 0 replies; 26+ messages in thread
From: Dilip Kumar @ 2023-12-11 05:39 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Ashutosh Bapat <[email protected]>; Postgres hackers <[email protected]>
On Tue, Nov 28, 2023 at 4:07 AM Michael Paquier <[email protected]> wrote:
>
> On Mon, Nov 27, 2023 at 12:14:05PM +0530, Ashutosh Bapat wrote:
> > Since you wroten "(still need to improve ...) I thought you are
> > working on v6. No problem. Sorry for the confusion.
>
> I see why my previous message could be confusing. Sorry about that.
I haven't specifically done a review or testing of this patch, but I
have used this for testing the CLOG group update code with my
SLRU-specific changes and I found it quite helpful to test some of the
concurrent areas where you need to stop processing somewhere in the
middle of the code and testing that area without this kind of
injection point framework is really difficult or may not be even
possible. We wanted to test the case of clog group update where we
can get multiple processes added to a single group and get the xid
status updated by the group leader, you can refer to my test in that
thread[1] (the last patch test_group_commit.patch is using this
framework for testing). Overall I feel this framework is quite useful
and easy to use as well.
[1] https://www.postgresql.org/message-id/CAFiTN-udSTGG_t5n9Z3eBbb4_%3DzNoKU%2B8FP-S6zpv-r4Gm-Y%2BQ%40ma...
--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 26+ messages in thread
end of thread, other threads:[~2023-12-11 05:39 UTC | newest]
Thread overview: 26+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2021-03-09 20:06 [PATCH] pg_dump: use a cursor in getBlobs.. Justin Pryzby <[email protected]>
2023-11-27 22:36 Re: Adding facility for injection points (or probe points?) for more advanced tests Michael Paquier <[email protected]>
2023-12-11 05:39 ` Re: Adding facility for injection points (or probe points?) for more advanced tests Dilip Kumar <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox