public inbox for [email protected]
help / color / mirror / Atom feed[PATCH] Shrink GISTSTATE
67+ messages / 11 participants
[nested] [flat]
* [PATCH] Shrink GISTSTATE
@ 2021-05-30 13:11 Andreas Karlsson <[email protected]>
0 siblings, 0 replies; 67+ messages in thread
From: Andreas Karlsson @ 2021-05-30 13:11 UTC (permalink / raw)
---
src/backend/access/gist/gist.c | 58 ++++++++++++++---------------
src/backend/access/gist/gistscan.c | 4 +-
src/backend/access/gist/gistsplit.c | 12 +++---
src/backend/access/gist/gistutil.c | 38 +++++++++----------
src/include/access/gist_private.h | 29 +++++++++------
5 files changed, 71 insertions(+), 70 deletions(-)
diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c
index 0683f42c25..8bccb76dd2 100644
--- a/src/backend/access/gist/gist.c
+++ b/src/backend/access/gist/gist.c
@@ -1507,11 +1507,6 @@ initGISTstate(Relation index)
MemoryContext oldCxt;
int i;
- /* safety check to protect fixed-size arrays in GISTSTATE */
- if (index->rd_att->natts > INDEX_MAX_KEYS)
- elog(ERROR, "numberOfAttributes %d > %d",
- index->rd_att->natts, INDEX_MAX_KEYS);
-
/* Create the memory context that will hold the GISTSTATE */
scanCxt = AllocSetContextCreate(CurrentMemoryContext,
"GiST scan context",
@@ -1519,7 +1514,8 @@ initGISTstate(Relation index)
oldCxt = MemoryContextSwitchTo(scanCxt);
/* Create and fill in the GISTSTATE */
- giststate = (GISTSTATE *) palloc(sizeof(GISTSTATE));
+ giststate = (GISTSTATE *) palloc(offsetof(GISTSTATE, column_state) +
+ sizeof(GIST_COL_STATE) * index->rd_att->natts);
giststate->scanCxt = scanCxt;
giststate->tempCxt = scanCxt; /* caller must change this if needed */
@@ -1541,54 +1537,54 @@ initGISTstate(Relation index)
for (i = 0; i < IndexRelationGetNumberOfKeyAttributes(index); i++)
{
- fmgr_info_copy(&(giststate->consistentFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].consistentFn),
index_getprocinfo(index, i + 1, GIST_CONSISTENT_PROC),
scanCxt);
- fmgr_info_copy(&(giststate->unionFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].unionFn),
index_getprocinfo(index, i + 1, GIST_UNION_PROC),
scanCxt);
/* opclasses are not required to provide a Compress method */
if (OidIsValid(index_getprocid(index, i + 1, GIST_COMPRESS_PROC)))
- fmgr_info_copy(&(giststate->compressFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].compressFn),
index_getprocinfo(index, i + 1, GIST_COMPRESS_PROC),
scanCxt);
else
- giststate->compressFn[i].fn_oid = InvalidOid;
+ giststate->column_state[i].compressFn.fn_oid = InvalidOid;
/* opclasses are not required to provide a Decompress method */
if (OidIsValid(index_getprocid(index, i + 1, GIST_DECOMPRESS_PROC)))
- fmgr_info_copy(&(giststate->decompressFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].decompressFn),
index_getprocinfo(index, i + 1, GIST_DECOMPRESS_PROC),
scanCxt);
else
- giststate->decompressFn[i].fn_oid = InvalidOid;
+ giststate->column_state[i].decompressFn.fn_oid = InvalidOid;
- fmgr_info_copy(&(giststate->penaltyFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].penaltyFn),
index_getprocinfo(index, i + 1, GIST_PENALTY_PROC),
scanCxt);
- fmgr_info_copy(&(giststate->picksplitFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].picksplitFn),
index_getprocinfo(index, i + 1, GIST_PICKSPLIT_PROC),
scanCxt);
- fmgr_info_copy(&(giststate->equalFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].equalFn),
index_getprocinfo(index, i + 1, GIST_EQUAL_PROC),
scanCxt);
/* opclasses are not required to provide a Distance method */
if (OidIsValid(index_getprocid(index, i + 1, GIST_DISTANCE_PROC)))
- fmgr_info_copy(&(giststate->distanceFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].distanceFn),
index_getprocinfo(index, i + 1, GIST_DISTANCE_PROC),
scanCxt);
else
- giststate->distanceFn[i].fn_oid = InvalidOid;
+ giststate->column_state[i].distanceFn.fn_oid = InvalidOid;
/* opclasses are not required to provide a Fetch method */
if (OidIsValid(index_getprocid(index, i + 1, GIST_FETCH_PROC)))
- fmgr_info_copy(&(giststate->fetchFn[i]),
+ fmgr_info_copy(&(giststate->column_state[i].fetchFn),
index_getprocinfo(index, i + 1, GIST_FETCH_PROC),
scanCxt);
else
- giststate->fetchFn[i].fn_oid = InvalidOid;
+ giststate->column_state[i].fetchFn.fn_oid = InvalidOid;
/*
* If the index column has a specified collation, we should honor that
@@ -1602,24 +1598,24 @@ initGISTstate(Relation index)
* any cases where a GiST storage type has a nondefault collation.)
*/
if (OidIsValid(index->rd_indcollation[i]))
- giststate->supportCollation[i] = index->rd_indcollation[i];
+ giststate->column_state[i].supportCollation = index->rd_indcollation[i];
else
- giststate->supportCollation[i] = DEFAULT_COLLATION_OID;
+ giststate->column_state[i].supportCollation = DEFAULT_COLLATION_OID;
}
/* No opclass information for INCLUDE attributes */
for (; i < index->rd_att->natts; i++)
{
- giststate->consistentFn[i].fn_oid = InvalidOid;
- giststate->unionFn[i].fn_oid = InvalidOid;
- giststate->compressFn[i].fn_oid = InvalidOid;
- giststate->decompressFn[i].fn_oid = InvalidOid;
- giststate->penaltyFn[i].fn_oid = InvalidOid;
- giststate->picksplitFn[i].fn_oid = InvalidOid;
- giststate->equalFn[i].fn_oid = InvalidOid;
- giststate->distanceFn[i].fn_oid = InvalidOid;
- giststate->fetchFn[i].fn_oid = InvalidOid;
- giststate->supportCollation[i] = InvalidOid;
+ giststate->column_state[i].consistentFn.fn_oid = InvalidOid;
+ giststate->column_state[i].unionFn.fn_oid = InvalidOid;
+ giststate->column_state[i].compressFn.fn_oid = InvalidOid;
+ giststate->column_state[i].decompressFn.fn_oid = InvalidOid;
+ giststate->column_state[i].penaltyFn.fn_oid = InvalidOid;
+ giststate->column_state[i].picksplitFn.fn_oid = InvalidOid;
+ giststate->column_state[i].equalFn.fn_oid = InvalidOid;
+ giststate->column_state[i].distanceFn.fn_oid = InvalidOid;
+ giststate->column_state[i].fetchFn.fn_oid = InvalidOid;
+ giststate->column_state[i].supportCollation = InvalidOid;
}
MemoryContextSwitchTo(oldCxt);
diff --git a/src/backend/access/gist/gistscan.c b/src/backend/access/gist/gistscan.c
index 61e92cf0f5..19fe161366 100644
--- a/src/backend/access/gist/gistscan.c
+++ b/src/backend/access/gist/gistscan.c
@@ -258,7 +258,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys,
* of function implementing filtering operator.
*/
fmgr_info_copy(&(skey->sk_func),
- &(so->giststate->consistentFn[skey->sk_attno - 1]),
+ &(so->giststate->column_state[skey->sk_attno - 1].consistentFn),
so->giststate->scanCxt);
/* Restore prior fn_extra pointers, if not first time */
@@ -304,7 +304,7 @@ gistrescan(IndexScanDesc scan, ScanKey key, int nkeys,
for (i = 0; i < scan->numberOfOrderBys; i++)
{
ScanKey skey = scan->orderByData + i;
- FmgrInfo *finfo = &(so->giststate->distanceFn[skey->sk_attno - 1]);
+ FmgrInfo *finfo = &(so->giststate->column_state[skey->sk_attno - 1].distanceFn);
/* Check we actually have a distance function ... */
if (!OidIsValid(finfo->fn_oid))
diff --git a/src/backend/access/gist/gistsplit.c b/src/backend/access/gist/gistsplit.c
index 526ed1218e..c8fc67f91c 100644
--- a/src/backend/access/gist/gistsplit.c
+++ b/src/backend/access/gist/gistsplit.c
@@ -378,16 +378,16 @@ genericPickSplit(GISTSTATE *giststate, GistEntryVector *entryvec, GIST_SPLITVEC
evec->n = v->spl_nleft;
memcpy(evec->vector, entryvec->vector + FirstOffsetNumber,
sizeof(GISTENTRY) * evec->n);
- v->spl_ldatum = FunctionCall2Coll(&giststate->unionFn[attno],
- giststate->supportCollation[attno],
+ v->spl_ldatum = FunctionCall2Coll(&giststate->column_state[attno].unionFn,
+ giststate->column_state[attno].supportCollation,
PointerGetDatum(evec),
PointerGetDatum(&nbytes));
evec->n = v->spl_nright;
memcpy(evec->vector, entryvec->vector + FirstOffsetNumber + v->spl_nleft,
sizeof(GISTENTRY) * evec->n);
- v->spl_rdatum = FunctionCall2Coll(&giststate->unionFn[attno],
- giststate->supportCollation[attno],
+ v->spl_rdatum = FunctionCall2Coll(&giststate->column_state[attno].unionFn,
+ giststate->column_state[attno].supportCollation,
PointerGetDatum(evec),
PointerGetDatum(&nbytes));
}
@@ -430,8 +430,8 @@ gistUserPicksplit(Relation r, GistEntryVector *entryvec, int attno, GistSplitVec
* Let the opclass-specific PickSplit method do its thing. Note that at
* this point we know there are no null keys in the entryvec.
*/
- FunctionCall2Coll(&giststate->picksplitFn[attno],
- giststate->supportCollation[attno],
+ FunctionCall2Coll(&giststate->column_state[attno].picksplitFn,
+ giststate->column_state[attno].supportCollation,
PointerGetDatum(entryvec),
PointerGetDatum(sv));
diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c
index 8dcd53c457..32a316b880 100644
--- a/src/backend/access/gist/gistutil.c
+++ b/src/backend/access/gist/gistutil.c
@@ -200,8 +200,8 @@ gistMakeUnionItVec(GISTSTATE *giststate, IndexTuple *itvec, int len,
}
/* Make union and store in attr array */
- attr[i] = FunctionCall2Coll(&giststate->unionFn[i],
- giststate->supportCollation[i],
+ attr[i] = FunctionCall2Coll(&giststate->column_state[i].unionFn,
+ giststate->column_state[i].supportCollation,
PointerGetDatum(evec),
PointerGetDatum(&attrsize));
@@ -269,8 +269,8 @@ gistMakeUnionKey(GISTSTATE *giststate, int attno,
}
*dstisnull = false;
- *dst = FunctionCall2Coll(&giststate->unionFn[attno],
- giststate->supportCollation[attno],
+ *dst = FunctionCall2Coll(&giststate->column_state[attno].unionFn,
+ giststate->column_state[attno].supportCollation,
PointerGetDatum(evec),
PointerGetDatum(&dstsize));
}
@@ -281,8 +281,8 @@ gistKeyIsEQ(GISTSTATE *giststate, int attno, Datum a, Datum b)
{
bool result;
- FunctionCall3Coll(&giststate->equalFn[attno],
- giststate->supportCollation[attno],
+ FunctionCall3Coll(&giststate->column_state[attno].equalFn,
+ giststate->column_state[attno].supportCollation,
a, b,
PointerGetDatum(&result));
return result;
@@ -554,12 +554,12 @@ gistdentryinit(GISTSTATE *giststate, int nkey, GISTENTRY *e,
gistentryinit(*e, k, r, pg, o, l);
/* there may not be a decompress function in opclass */
- if (!OidIsValid(giststate->decompressFn[nkey].fn_oid))
+ if (!OidIsValid(giststate->column_state[nkey].decompressFn.fn_oid))
return;
dep = (GISTENTRY *)
- DatumGetPointer(FunctionCall1Coll(&giststate->decompressFn[nkey],
- giststate->supportCollation[nkey],
+ DatumGetPointer(FunctionCall1Coll(&giststate->column_state[nkey].decompressFn,
+ giststate->column_state[nkey].supportCollation,
PointerGetDatum(e)));
/* decompressFn may just return the given pointer */
if (dep != e)
@@ -612,10 +612,10 @@ gistCompressValues(GISTSTATE *giststate, Relation r,
gistentryinit(centry, attdata[i], r, NULL, (OffsetNumber) 0,
isleaf);
/* there may not be a compress function in opclass */
- if (OidIsValid(giststate->compressFn[i].fn_oid))
+ if (OidIsValid(giststate->column_state[i].compressFn.fn_oid))
cep = (GISTENTRY *)
- DatumGetPointer(FunctionCall1Coll(&giststate->compressFn[i],
- giststate->supportCollation[i],
+ DatumGetPointer(FunctionCall1Coll(&giststate->column_state[i].compressFn,
+ giststate->column_state[i].supportCollation,
PointerGetDatum(¢ry)));
else
cep = ¢ry;
@@ -650,8 +650,8 @@ gistFetchAtt(GISTSTATE *giststate, int nkey, Datum k, Relation r)
gistentryinit(fentry, k, r, NULL, (OffsetNumber) 0, false);
fep = (GISTENTRY *)
- DatumGetPointer(FunctionCall1Coll(&giststate->fetchFn[nkey],
- giststate->supportCollation[nkey],
+ DatumGetPointer(FunctionCall1Coll(&giststate->column_state[nkey].fetchFn,
+ giststate->column_state[nkey].supportCollation,
PointerGetDatum(&fentry)));
/* fetchFn set 'key', return it to the caller */
@@ -676,14 +676,14 @@ gistFetchTuple(GISTSTATE *giststate, Relation r, IndexTuple tuple)
datum = index_getattr(tuple, i + 1, giststate->leafTupdesc, &isnull[i]);
- if (giststate->fetchFn[i].fn_oid != InvalidOid)
+ if (giststate->column_state[i].fetchFn.fn_oid != InvalidOid)
{
if (!isnull[i])
fetchatt[i] = gistFetchAtt(giststate, i, datum, r);
else
fetchatt[i] = (Datum) 0;
}
- else if (giststate->compressFn[i].fn_oid == InvalidOid)
+ else if (giststate->column_state[i].compressFn.fn_oid == InvalidOid)
{
/*
* If opclass does not provide compress method that could change
@@ -726,11 +726,11 @@ gistpenalty(GISTSTATE *giststate, int attno,
{
float penalty = 0.0;
- if (giststate->penaltyFn[attno].fn_strict == false ||
+ if (giststate->column_state[attno].penaltyFn.fn_strict == false ||
(isNullOrig == false && isNullAdd == false))
{
- FunctionCall3Coll(&giststate->penaltyFn[attno],
- giststate->supportCollation[attno],
+ FunctionCall3Coll(&giststate->column_state[attno].penaltyFn,
+ giststate->column_state[attno].supportCollation,
PointerGetDatum(orig),
PointerGetDatum(add),
PointerGetDatum(&penalty));
diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h
index 553d364e2d..7482482e42 100644
--- a/src/include/access/gist_private.h
+++ b/src/include/access/gist_private.h
@@ -59,6 +59,22 @@ typedef struct
#define PAGE_NO_SPACE(nbp, itup) (PAGE_FREE_SPACE(nbp) < \
MAXALIGN(IndexTupleSize(itup)))
+typedef struct GIST_COL_STATE
+{
+ FmgrInfo consistentFn;
+ FmgrInfo unionFn;
+ FmgrInfo compressFn;
+ FmgrInfo decompressFn;
+ FmgrInfo penaltyFn;
+ FmgrInfo picksplitFn;
+ FmgrInfo equalFn;
+ FmgrInfo distanceFn;
+ FmgrInfo fetchFn;
+
+ /* Collations to pass to the support functions */
+ Oid supportCollation;
+} GIST_COL_STATE;
+
/*
* GISTSTATE: information needed for any GiST index operation
*
@@ -83,18 +99,7 @@ typedef struct GISTSTATE
TupleDesc fetchTupdesc; /* tuple descriptor for tuples returned in an
* index-only scan */
- FmgrInfo consistentFn[INDEX_MAX_KEYS];
- FmgrInfo unionFn[INDEX_MAX_KEYS];
- FmgrInfo compressFn[INDEX_MAX_KEYS];
- FmgrInfo decompressFn[INDEX_MAX_KEYS];
- FmgrInfo penaltyFn[INDEX_MAX_KEYS];
- FmgrInfo picksplitFn[INDEX_MAX_KEYS];
- FmgrInfo equalFn[INDEX_MAX_KEYS];
- FmgrInfo distanceFn[INDEX_MAX_KEYS];
- FmgrInfo fetchFn[INDEX_MAX_KEYS];
-
- /* Collations to pass to the support functions */
- Oid supportCollation[INDEX_MAX_KEYS];
+ GIST_COL_STATE column_state[FLEXIBLE_ARRAY_MEMBER];
} GISTSTATE;
--
2.30.2
--------------7F3894F7B390FDB09EFDB0AE--
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
@ 2022-03-04 16:10 ` Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
1 sibling, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-03-04 16:10 UTC (permalink / raw)
To: Georgios <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>
The patch is failing on cfbot/freebsd.
http://cfbot.cputube.org/georgios-kokolatos.html
Also, I wondered if you'd looked at the "high compression" interfaces in
lz4hc.h ? Should pg_dump use that ?
On Fri, Feb 25, 2022 at 08:03:40AM -0600, Justin Pryzby wrote:
> Thanks for working on this. Your 0001 looks similar to what I did for zstd 1-2
> years ago.
> https://commitfest.postgresql.org/32/2888/
>
> I rebased and attached the latest patches I had in case they're useful to you.
> I'd like to see zstd included in pg_dump eventually, but it was too much work
> to shepherd the patches. Now that seems reasonable for pg16.
>
> With the other compression patches I've worked on, we've used an extra patch
> with changes the default to the new compression algorithm, to force cfbot to
> exercize the new code.
>
> Do you know the process with commitfests and cfbot ?
> There's also this, which allows running the tests on cirrus before mailing the
> patch to the hackers list.
> ./src/tools/ci/README
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-25 05:20 ` Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Greg Stark @ 2022-03-25 05:20 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
It seems development on this has stalled. If there's no further work
happening I guess I'll mark the patch returned with feedback. Feel
free to resubmit it to the next CF when there's progress.
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
@ 2022-03-25 13:22 ` Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-03-25 13:22 UTC (permalink / raw)
To: Greg Stark <[email protected]>; +Cc: Georgios <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Fri, Mar 25, 2022 at 01:20:47AM -0400, Greg Stark wrote:
> It seems development on this has stalled. If there's no further work
> happening I guess I'll mark the patch returned with feedback. Feel
> free to resubmit it to the next CF when there's progress.
Since it's a reasonably large patch (and one that I had myself started before)
and it's only been 20some days since (minor) review comments, and since the
focus right now is on committing features, and not reviewing new patches, and
this patch is new one month ago, and its 0002 not intended for pg15, therefor
I'm moving it to the next CF, where I hope to work with its authors to progress
it.
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-25 23:13 ` Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Rachel Heaton @ 2022-03-25 23:13 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Greg Stark <[email protected]>; Georgios <[email protected]>; Postgres hackers <[email protected]>
On Fri, Mar 25, 2022 at 6:22 AM Justin Pryzby <[email protected]> wrote:
>
> On Fri, Mar 25, 2022 at 01:20:47AM -0400, Greg Stark wrote:
> > It seems development on this has stalled. If there's no further work
> > happening I guess I'll mark the patch returned with feedback. Feel
> > free to resubmit it to the next CF when there's progress.
>
> Since it's a reasonably large patch (and one that I had myself started before)
> and it's only been 20some days since (minor) review comments, and since the
> focus right now is on committing features, and not reviewing new patches, and
> this patch is new one month ago, and its 0002 not intended for pg15, therefor
> I'm moving it to the next CF, where I hope to work with its authors to progress
> it.
>
Hi Folks,
Here is an updated patchset from Georgios, with minor assistance from myself.
The comments above should be addressed, but please let us know if
there are other things to go over. A functional change in this
patchset is when `--compress=none` is passed to pg_dump, it will not
compress for directory type (previously, it would use gzip if
present). The previous default behavior is retained.
- Rachel
Attachments:
[application/octet-stream] v2-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch (5.4K, ../../CADJcwiUbfBV0oNNpKvd2H4kP7m2U+7SmL-GAGO8Uv5H3HZUbAw@mail.gmail.com/2-v2-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch)
download | inline diff:
From 67480e216a201dfde9aa2dc48b39b37e39bf46fb Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 25 Mar 2022 14:00:28 +0000
Subject: [PATCH v2 1/4] Extend compression coverage for pg_dump, pg_restore
---
src/bin/pg_dump/t/001_basic.pl | 15 ++++++
src/bin/pg_dump/t/002_pg_dump.pl | 92 ++++++++++++++++++++++++++++++++
2 files changed, 107 insertions(+)
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 48faeb4371..8fdacea6f5 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -126,6 +126,21 @@ command_fails_like(
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
'pg_dump: -Z/--compress must be in range');
+if (check_pg_config("#define HAVE_LIBZ 1"))
+{
+ command_fails_like(
+ [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
+ 'pg_dump: compression is not supported by tar archive format');
+}
+else
+{
+ command_fails_like(
+ [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
+ 'pg_dump: warning: requested compression not available in this installation -- archive will be uncompressed');
+}
+
command_fails_like(
[ 'pg_dump', '--extra-float-digits', '-16' ],
qr/\Qpg_dump: error: --extra-float-digits must be in range\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 3e55ff26f8..cbf4524503 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -55,6 +55,81 @@ my %pgdump_runs = (
"$tempdir/binary_upgrade.dump",
],
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=custom', '--compress=1',
+ "--file=$tempdir/compression_gzip_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_custom_format.sql",
+ "$tempdir/compression_gzip_custom_format.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=directory', '--compress=1',
+ "--file=$tempdir/compression_gzip_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_directory_format.sql",
+ "$tempdir/compression_gzip_directory_format",
+ ],
+ },
+
+ compression_gzip_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=6',
+ "--file=$tempdir/compression_gzip_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ '--jobs=3',
+ "--file=$tempdir/compression_gzip_directory_format_parallel.sql",
+ "$tempdir/compression_gzip_directory_format_parallel",
+ ],
+ },
+
+ # Check that the output is valid gzip
+ compression_gzip_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '-Z1',
+ "--file=$tempdir/compression_gzip_plain_format.sql.gz",
+ 'postgres',
+ ],
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-k', '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -425,6 +500,7 @@ my %full_runs = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3007,6 +3083,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_test_table => 1,
@@ -3080,6 +3157,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3856,9 +3934,23 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
my $run_db = 'postgres';
+ my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
+ my $compress_program = $ENV{GZIP_PROGRAM};
+
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
+ if (defined($pgdump_runs{$run}->{compress_cmd}))
+ {
+ # Skip compression_cmd tests when compression is not supported,
+ # as the result is uncompressed or the utility program does not
+ # exist
+ next if !$supports_compression || !defined($compress_program)
+ || $compress_program eq '';
+ command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
+ "$run: compression commands");
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.33.0
[application/octet-stream] v2-0002-Remove-unsupported-bitrot-compression-from-pg_dum.patch (4.1K, ../../CADJcwiUbfBV0oNNpKvd2H4kP7m2U+7SmL-GAGO8Uv5H3HZUbAw@mail.gmail.com/3-v2-0002-Remove-unsupported-bitrot-compression-from-pg_dum.patch)
download | inline diff:
From fdd7d2087ade23bc5e729b478fb524f4d63776f9 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 25 Mar 2022 14:56:50 +0000
Subject: [PATCH v2 2/4] Remove unsupported/bitrot compression from pg_dump tar
---
src/bin/pg_dump/pg_backup_tar.c | 82 ++++-----------------------------
1 file changed, 9 insertions(+), 73 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 5c351acda0..a31376bfa8 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -65,11 +65,6 @@ static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
typedef struct
{
-#ifdef HAVE_LIBZ
- gzFile zFH;
-#else
- FILE *zFH;
-#endif
FILE *nFH;
FILE *tarFH;
FILE *tmpFH;
@@ -248,14 +243,7 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
ctx->filename = pg_strdup(fn);
}
else
@@ -320,10 +308,6 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
lclContext *ctx = (lclContext *) AH->formatData;
TAR_MEMBER *tm;
-#ifdef HAVE_LIBZ
- char fmode[14];
-#endif
-
if (mode == 'r')
{
tm = _tarPositionTo(AH, filename);
@@ -344,16 +328,10 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
-#ifdef HAVE_LIBZ
-
if (AH->compression == 0)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
- /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
-#else
- tm->nFH = ctx->tarFH;
-#endif
}
else
{
@@ -405,21 +383,10 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
-#ifdef HAVE_LIBZ
-
- if (AH->compression != 0)
- {
- sprintf(fmode, "wb%d", AH->compression);
- tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
- if (tm->zFH == NULL)
- fatal("could not open temporary file");
- }
- else
+ if (AH->compression == 0)
tm->nFH = tm->tmpFH;
-#else
-
- tm->nFH = tm->tmpFH;
-#endif
+ else
+ fatal("compression is not supported by tar archive format");
tm->AH = AH;
tm->targetFile = pg_strdup(filename);
@@ -434,15 +401,8 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- /*
- * Close the GZ file since we dup'd. This will flush the buffers.
- */
if (AH->compression != 0)
- {
- errno = 0; /* in case gzclose() doesn't set it */
- if (GZCLOSE(th->zFH) != 0)
- fatal("could not close tar member: %m");
- }
+ fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
_tarAddFile(AH, th); /* This will close the temp file */
@@ -456,7 +416,6 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
free(th->targetFile);
th->nFH = NULL;
- th->zFH = NULL;
}
#ifdef __NOT_USED__
@@ -542,29 +501,9 @@ _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
}
else if (th)
{
- if (th->zFH)
- {
- res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
- if (res != len && !GZEOF(th->zFH))
- {
-#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(th->zFH, &errnum);
-
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
-#else
- fatal("could not read from input file: %s",
- strerror(errno));
-#endif
- }
- }
- else
- {
- res = fread(&((char *) buf)[used], 1, len, th->nFH);
- if (res != len && !feof(th->nFH))
- READ_ERROR_EXIT(th->nFH);
- }
+ res = fread(&((char *) buf)[used], 1, len, th->nFH);
+ if (res != len && !feof(th->nFH))
+ READ_ERROR_EXIT(th->nFH);
}
}
@@ -596,10 +535,7 @@ tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
{
size_t res;
- if (th->zFH != NULL)
- res = GZWRITE(buf, 1, len, th->zFH);
- else
- res = fwrite(buf, 1, len, th->nFH);
+ res = fwrite(buf, 1, len, th->nFH);
th->pos += res;
return res;
--
2.33.0
[application/octet-stream] v2-0003-Prepare-pg_dump-for-additional-compression-method.patch (54.0K, ../../CADJcwiUbfBV0oNNpKvd2H4kP7m2U+7SmL-GAGO8Uv5H3HZUbAw@mail.gmail.com/4-v2-0003-Prepare-pg_dump-for-additional-compression-method.patch)
download | inline diff:
From f7d10ce401a9b7bea2ecff6e4e739b733e74f3ae Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 25 Mar 2022 15:12:41 +0000
Subject: [PATCH v2 3/4] Prepare pg_dump for additional compression methods
This commmit does the heavy lifting required for additional compression methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
Furthermore, compression was chosen based on the value of the level passed
as an argument during the invocation of pg_dump or some hardcoded defaults. This
does not scale for more than one compression methods. Now the method used for
compression can be explicitly requested during command invocation, or set during
hardcoded defaults. Then it is stored in the relevant structs and passed in the
relevant functions, along side compression level which has lost it's special
meaning. The method for compression is not yet stored in the actual archive.
This is done in the next commit which does introduce a new method.
The previously named CompressionAlgorithm enum is changed for
CompressionMethod so that it matches better similar variables found through out
the code base.
In a fashion similar to the binary for pg_basebackup, the method for compression
is passed using the already existing -Z/--compress parameter of pg_dump. The
legacy format and behaviour is maintained. Additionally, the user can explicitly
pass a requested method and optionaly the level to be used after a semicolon,
e.g. --compress=gzip:6
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 425 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 32 +-
src/bin/pg_dump/pg_backup.h | 14 +-
src/bin/pg_dump/pg_backup_archiver.c | 171 +++++------
src/bin/pg_dump/pg_backup_archiver.h | 46 +--
src/bin/pg_dump/pg_backup_custom.c | 11 +-
src/bin/pg_dump/pg_backup_directory.c | 12 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 155 ++++++++--
src/bin/pg_dump/t/001_basic.pl | 14 +-
src/bin/pg_dump/t/002_pg_dump.pl | 50 ++-
src/tools/pgindent/typedefs.list | 2 +-
14 files changed, 616 insertions(+), 360 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2f0042fd96..992b7312df 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 84306d94ee..56e041c9b3 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
+export GZIP_PROGRAM=$(GZIP)
+
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 9077fdb74d..bab723c84f 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ CompressionMethod compressionMethod;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compressionMethod = compressionMethod;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (compressionMethod == COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compressionLevel);
#endif
return cs;
@@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
+ int compressionLevel, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +152,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compressionMethod)
{
- case COMPR_ALG_LIBZ:
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compressionMethod)
+ {
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ CompressionMethod compressionMethod;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -455,18 +441,18 @@ cfopen_read(const char *path, const char *mode)
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
#endif
@@ -479,26 +465,31 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * When 'compressionMethod' indicates gzip, a gzip compressed stream is opened,
+ * and 'compressionLevel' is used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
+ *
+ * It is the caller's responsibility to verify that the requested
+ * 'compressionMethod' is supported by the build.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compressionMethod == COMPRESSION_NONE)
+ fp = cfopen(path, mode, compressionMethod, 0);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
free_keep_errno(fname);
#else
fatal("not built with zlib support");
@@ -509,60 +500,94 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compressionMethod' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ CompressionMethod compressionMethod, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compressionMethod = compressionMethod;
+
+ switch (compressionMethod)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(path, -1, mode, compressionMethod, compressionLevel);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +597,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +659,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- fatal("could not read from input file: %s", strerror(errno));
- else
- fatal("could not read from input file: end of file");
- }
- }
- else
+ case COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile)fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +692,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compressionMethod)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compressionMethod == COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..b8b366616c 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -17,16 +17,17 @@
#include "pg_backup_archiver.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#else
+/* this is just the redefinition of a libz constant */
+#define Z_DEFAULT_COMPRESSION (-1)
+#endif
+
/* Initial buffer sizes used in zlib compression. */
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +47,12 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ CompressionMethod compressionMethod,
+ int compressionLevel,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +61,16 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+extern cfp *cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index fcc5f6bd05..d9f6c9a087 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -75,6 +75,13 @@ enum _dumpPreparedQueries
NUM_PREP_QUERIES /* must be last */
};
+typedef enum _compressionMethod
+{
+ COMPRESSION_GZIP,
+ COMPRESSION_NONE,
+ COMPRESSION_INVALID
+} CompressionMethod;
+
/* Parameters needed by ConnectDatabase; same for dump and restore */
typedef struct _connParams
{
@@ -143,7 +150,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ CompressionMethod compressionMethod;
+ int compressionLevel;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +311,9 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compression,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index d41a99d6ea..7d96446f1a 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,9 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +94,11 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +237,15 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt,
+ compressionMethod, compressionLevel,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +255,8 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, COMPRESSION_NONE, 0, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
@@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compressionMethod == COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +459,9 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename,
+ ropt->compressionMethod, ropt->compressionLevel);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -740,7 +741,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -970,6 +971,7 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compressionMethod = COMPRESSION_NONE;
return opts;
}
@@ -1117,13 +1119,13 @@ PrintTOCSummary(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, COMPRESSION_NONE, 0);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1132,7 +1134,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compressionLevel);
switch (AH->format)
{
@@ -1486,60 +1488,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod, int compressionLevel)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compressionMethod, compressionLevel);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compressionMethod, compressionLevel);
if (!AH->OF)
{
@@ -1550,33 +1527,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *)AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1700,22 +1668,16 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
-
- else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're
+ * connected then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ else
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2200,7 +2162,9 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2251,14 +2215,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compressionMethod = compressionMethod;
+ AH->compressionLevel = compressionLevel;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, COMPRESSION_NONE, 0);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2266,7 +2230,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compressionMethod != COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3716,7 +3680,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compressionLevel);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3790,18 +3754,21 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compressionLevel = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compressionLevel = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compressionLevel = Z_DEFAULT_COMPRESSION;
+ if (AH->compressionLevel != INT_MIN)
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+#else
+ AH->compressionMethod = COMPRESSION_GZIP;
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 540d4f6a83..837e9d73f5 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,17 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ CompressionMethod compressionMethod; /* Requested method for compression */
+ int compressionLevel; /*---------
+ * Requested level of compression for method.
+ * Possible values for compression:
+ * INT_MIN when no compression method is
+ * requested.
+ * -1 Z_DEFAULT_COMPRESSION for gzip
+ * compression.
+ * 1-9 levels for gzip compression.
+ *---------
+ */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..7f38ea9cd5 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -377,7 +379,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +570,8 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compressionMethod, AH->compressionLevel,
+ _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7f4e340dea..0e60b447de 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod,
+ AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W,
+ COMPRESSION_NONE, 0);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -644,7 +647,7 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", COMPRESSION_NONE, 0);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +665,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod, AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index a31376bfa8..bd99b3826a 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -801,7 +802,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -890,7 +890,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
sfx = ".gz";
else
sfx = "";
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index e69dcf8a48..cd91efdd7a 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -55,6 +55,7 @@
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
#include "common/connect.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -163,6 +164,9 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression_option(const char *opt,
+ CompressionMethod *compressionMethod,
+ int *compressLevel);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -336,8 +340,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ int compressLevel = INT_MIN;
+ CompressionMethod compressionMethod = COMPRESSION_INVALID;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -557,9 +562,9 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression_option(optarg, &compressionMethod,
+ &compressLevel))
exit_nicely(1);
break;
@@ -689,23 +694,21 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /* Set default compressionMethod unless one already set by the user */
+ if (compressionMethod == COMPRESSION_INVALID)
{
+ compressionMethod = COMPRESSION_NONE;
+
#ifdef HAVE_LIBZ
+ /* Custom and directory formats are compressed by default (zlib) */
if (archiveFormat == archCustom || archiveFormat == archDirectory)
+ {
+ compressionMethod = COMPRESSION_GZIP;
compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ }
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -718,8 +721,9 @@ main(int argc, char **argv)
fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat,
+ compressionMethod, compressLevel,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -950,10 +954,8 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compressionLevel = compressLevel;
+ ropt->compressionMethod = compressionMethod;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -1000,7 +1002,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ " compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1260,6 +1263,116 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+static bool
+parse_compression_method(const char *method,
+ CompressionMethod *compressionMethod)
+{
+ bool res = true;
+
+ if (pg_strcasecmp(method, "gzip") == 0)
+ *compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "none") == 0)
+ *compressionMethod = COMPRESSION_NONE;
+ else
+ {
+ pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ res = false;
+ }
+
+ return res;
+}
+
+/*
+ * Interprets a compression option of the format 'method[:LEVEL]' of legacy just
+ * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level
+ * are returned in *compressionMethod and *compressionLevel. In case of error,
+ * the function returns false and then the values of *compression{Method,Level}
+ * are not to be trusted.
+ */
+static bool
+parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
+ int *compressLevel)
+{
+ char *method;
+ const char *sep;
+ int methodlen;
+ bool supports_compression = true;
+ bool res = true;
+
+ /* find the separator if exists */
+ sep = strchr(opt, ':');
+
+ /*
+ * If there is no separator, then it is either a legacy format, or only the
+ * method has been passed.
+ */
+ if (!sep)
+ {
+ if (strspn(opt, "-0123456789") == strlen(opt))
+ {
+ res = option_parse_int(opt, "-Z/--compress", 0, 9, compressLevel);
+ *compressionMethod = (*compressLevel > 0) ? COMPRESSION_GZIP :
+ COMPRESSION_NONE;
+ }
+ else
+ res = parse_compression_method(opt, compressionMethod);
+ }
+ else
+ {
+ /* otherwise, it should be method:LEVEL */
+ methodlen = sep - opt + 1;
+ method = pg_malloc0(methodlen);
+ snprintf(method, methodlen, "%.*s", methodlen - 1, opt);
+
+ res = parse_compression_method(method, compressionMethod);
+ if (res)
+ {
+ sep++;
+ if (*sep == '\0')
+ {
+ pg_log_error("no level defined for compression \"%s\"", method);
+ pg_free(method);
+ res = false;
+ }
+ else
+ {
+ res = option_parse_int(sep, "-Z/--compress [LEVEL]", 1, 9,
+ compressLevel);
+ }
+ }
+ }
+
+ /* if there is an error, there is no need to check further */
+ if (!res)
+ return res;
+
+ /* one can set level when method is gzip */
+ if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ {
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ return false;
+ }
+
+ /* verify that the requested compression is supported */
+#ifndef HAVE_LIBZ
+ if (*compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (*compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ *compressionMethod = COMPRESSION_NONE;
+ *compressLevel = INT_MIN;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 8fdacea6f5..f6bccfe55b 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,6 +121,16 @@ command_fails_like(
qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/,
'pg_restore: cannot specify both --single-transaction and multiple jobs');
+command_fails_like(
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ 'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
+
command_fails_like(
[ 'pg_dump', '-Z', '-1' ],
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
@@ -129,14 +139,14 @@ command_fails_like(
if (check_pg_config("#define HAVE_LIBZ 1"))
{
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
}
else
{
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: requested compression not available in this installation -- archive will be uncompressed');
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index cbf4524503..49f04c9477 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -77,7 +77,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_directory_format",
'postgres',
],
@@ -98,7 +98,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=6',
+ '--format=directory', '--compress=gzip:6',
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
@@ -130,6 +130,25 @@ my %pgdump_runs = (
"$tempdir/compression_gzip_plain_format.sql.gz",
],
},
+ compression_none_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ '--compress=none',
+ "--file=$tempdir/compression_none_dir_format",
+ 'postgres',
+ ],
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_none_dir_format.sql",
+ "$tempdir/compression_none_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -229,7 +248,7 @@ my %pgdump_runs = (
defaults_dir_format => {
test_key => 'defaults',
dump_cmd => [
- 'pg_dump', '-Fd',
+ 'pg_dump', '-Fd',
"--file=$tempdir/defaults_dir_format", 'postgres',
],
restore_cmd => [
@@ -3950,6 +3969,31 @@ foreach my $run (sort keys %pgdump_runs)
command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
"$run: compression commands");
}
+ if (defined($pgdump_runs{$run}->{glob_match}))
+ {
+ # Skip compression_cmd tests when compression is not supported,
+ # as the result is uncompressed or the utility program does not
+ # exist
+ next if !$supports_compression || !defined($compress_program)
+ || $compress_program eq '';
+
+ my $match = $pgdump_runs{$run}->{glob_match}->{match};
+ my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
+ $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
+ my @glob_matched = glob $match;
+
+ cmp_ok(scalar(@glob_matched), '>=', $match_count,
+ "Expected at least $match_count file(s) matching $match");
+
+ if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ {
+ my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
+ my @glob_matched = glob $no_match;
+
+ cmp_ok(scalar(@glob_matched), '==', 0,
+ "Expected no file(s) matching $no_match");
+ }
+ }
if ($pgdump_runs{$run}->{restore_cmd})
{
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d9b83f744f..7cdfe09e46 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -411,7 +411,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
-CompressionAlgorithm
+CompressionMethod
CompressorState
ComputeXidHorizonsResult
ConditionVariable
--
2.33.0
[application/octet-stream] v2-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (46.3K, ../../CADJcwiUbfBV0oNNpKvd2H4kP7m2U+7SmL-GAGO8Uv5H3HZUbAw@mail.gmail.com/5-v2-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 5478e07e1c99db8403dccacab7e2f7768c498eac Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 25 Mar 2022 16:34:50 +0000
Subject: [PATCH v2 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_io.{c,h} there are two distinct APIs exposed, the streaming API
and a file API. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
In the later case, the API is using an opaque wrapper around a file stream,
which aquired via fopen() or gzopen() respectively. It would then provide
wrappers around fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(); or
their gz equivallents. However the LZ4F api does not provide this functionality.
So this has been implemented localy.
In order to maintain the API compatibility a new structure LZ4File is
introduced. It is responsible for keeping state and any yet unused generated
content. The later is required when the generated decompressed output, exceeds
the caller's buffer capacity.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression is
still stored in the dump, though admittedly is of no apparent use.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_io.c | 731 ++++++++++++++++++++++++---
src/bin/pg_dump/pg_backup.h | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 71 ++-
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_dump.c | 12 +-
src/bin/pg_dump/t/001_basic.pl | 4 +-
src/bin/pg_dump/t/002_pg_dump.pl | 206 ++++++--
9 files changed, 905 insertions(+), 147 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 992b7312df..310cb094da 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression will be used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 56e041c9b3..625c00b123 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index bab723c84f..8c372ff196 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -56,6 +58,14 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBLZ4
+#include "lz4.h"
+#include "lz4frame.h"
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -69,9 +79,9 @@ struct CompressorState
#ifdef HAVE_LIBZ
z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
#endif
+ void *outbuf;
+ size_t outsize;
};
/* Routines that support zlib compressed data I/O */
@@ -85,6 +95,15 @@ static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
#endif
+/* Routines that support LZ4 compressed data I/O */
+#ifdef HAVE_LIBLZ4
+static void InitCompressorLZ4(CompressorState *cs);
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+#endif
+
/* Routines that support uncompressed data I/O */
static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
@@ -103,6 +122,11 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
+#ifndef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ fatal("not built with LZ4 support");
+#endif
+
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
@@ -115,6 +139,10 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
InitCompressorZlib(cs, compressionLevel);
#endif
+#ifdef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ InitCompressorLZ4(cs);
+#endif
return cs;
}
@@ -137,6 +165,13 @@ ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
ReadDataFromArchiveZlib(AH, readF);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ReadDataFromArchiveLZ4(AH, readF);
+#else
+ fatal("not built with lz4 support");
#endif
break;
default:
@@ -159,6 +194,13 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ WriteDataToArchiveLZ4(AH, cs, data, dLen);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -183,6 +225,13 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
EndCompressorZlib(AH, cs);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ EndCompressorLZ4(AH, cs);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -213,20 +262,20 @@ InitCompressorZlib(CompressorState *cs, int level)
zp->opaque = Z_NULL;
/*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
+ * outsize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to append a
* trailing zero byte to the zlib output.
*/
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
+ cs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ cs->outsize = ZLIB_OUT_SIZE;
if (deflateInit(zp, level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
/* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = cs->outbuf;
+ zp->avail_out = cs->outsize;
}
static void
@@ -243,7 +292,7 @@ EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
if (deflateEnd(zp) != Z_OK)
fatal("could not close compression stream: %s", zp->msg);
- free(cs->zlibOut);
+ free(cs->outbuf);
free(cs->zp);
}
@@ -251,7 +300,7 @@ static void
DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
{
z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
+ void *out = cs->outbuf;
int res = Z_OK;
while (cs->zp->avail_in != 0 || flush)
@@ -259,7 +308,7 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
if (res == Z_STREAM_ERROR)
fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
+ if ((flush && (zp->avail_out < cs->outsize))
|| (zp->avail_out == 0)
|| (zp->avail_in != 0)
)
@@ -269,18 +318,18 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
* chunk is the EOF marker in the custom format. This should never
* happen but...
*/
- if (zp->avail_out < cs->zlibOutSize)
+ if (zp->avail_out < cs->outsize)
{
/*
* Any write function should do its own error checking but to
* make sure we do a check here as well...
*/
- size_t len = cs->zlibOutSize - zp->avail_out;
+ size_t len = cs->outsize - zp->avail_out;
- cs->writeF(AH, out, len);
+ cs->writeF(AH, (char *)out, len);
}
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = out;
+ zp->avail_out = cs->outsize;
}
if (res == Z_STREAM_END)
@@ -364,6 +413,71 @@ ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
}
#endif /* HAVE_LIBZ */
+#ifdef HAVE_LIBLZ4
+static void
+InitCompressorLZ4(CompressorState *cs)
+{
+ /* Will be lazy init'd */
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ pg_free(cs->outbuf);
+
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = (4 * 1024) + 1;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen)
+{
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > cs->outsize)
+ {
+ cs->outbuf = pg_realloc(cs->outbuf, requiredsize);
+ cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, cs->outbuf,
+ dLen, cs->outsize);
+
+ if (compressed <= 0)
+ fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, cs->outbuf, compressed);
+}
+#endif /* HAVE_LIBLZ4 */
/*
* Functions for uncompressed output.
@@ -400,9 +514,36 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*----------------------
*/
+#ifdef HAVE_LIBLZ4
/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
+ * State needed for LZ4 (de)compression using the cfp API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+#endif
+
+/*
+ * cfp represents an open stream, wrapping the underlying FILE, gzFile
+ * pointer, or LZ4File pointer. This is opaque to the callers.
*/
struct cfp
{
@@ -410,9 +551,7 @@ struct cfp
void *fp;
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -424,26 +563,380 @@ free_keep_errno(void *p)
errno = save_errno;
}
+#ifdef HAVE_LIBLZ4
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(LZ4File *fs)
+{
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_error(LZ4File *fs)
+{
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the necessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File *fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen, &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File *fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ readlen = p - fs->overflowbuf + 1; /* Include the line terminating char */
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read(LZ4File *fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *)readbuf;
+ rend = (char *)readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr
+ * if the eol flag is set, either skip if one already found or fill up to EOL
+ * if present in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size -1 -dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true && (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t)(p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *)ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf, fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int)dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static int
+LZ4File_write(LZ4File *fs, const void *ptr, int size)
+{
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fgetc() and gzgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(LZ4File *fs)
+{
+ unsigned char c;
+
+ if (LZ4File_read(fs, &c, 1, false) != 1)
+ return EOF;
+
+ return c;
+}
+
+/*
+ * fgets() and gzgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(LZ4File *fs, char *buf, int len)
+{
+ size_t dsize;
+
+ dsize = LZ4File_read(fs, buf, len, true);
+ if (dsize < 0)
+ fatal("failed to read from archive %s", LZ4File_error(fs));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return buf;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(LZ4File *fs)
+{
+ FILE *fp;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ fatal("failed to end decompression: %s", LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+#endif
+
/*
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * this will open either "foo" or "foo.gz" or "foo.lz4", trying in that order.
*
* On failure, return NULL with an error code in errno.
+ *
*/
cfp *
cfopen_read(const char *path, const char *mode)
{
- cfp *fp;
+ cfp *fp = NULL;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
+ else if (hasSuffix(path, ".lz4"))
+ fp = cfopen(path, mode, COMPRESSION_LZ4, 0);
else
-#endif
{
fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
@@ -455,8 +948,19 @@ cfopen_read(const char *path, const char *mode)
fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
+#endif
+#ifdef HAVE_LIBLZ4
+ if (fp == NULL)
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, COMPRESSION_LZ4, 0);
+ free_keep_errno(fname);
+ }
#endif
}
+
return fp;
}
@@ -467,7 +971,8 @@ cfopen_read(const char *path, const char *mode)
*
* When 'compressionMethod' indicates gzip, a gzip compressed stream is opened,
* and 'compressionLevel' is used. The ".gz" suffix is automatically added to
- * 'path' in that case.
+ * 'path' in that case. The same applies when 'compressionMethod' indicates lz4,
+ * but then the ".lz4" suffix is added instead.
*
* It is the caller's responsibility to verify that the requested
* 'compressionMethod' is supported by the build.
@@ -479,23 +984,44 @@ cfopen_write(const char *path, const char *mode,
CompressionMethod compressionMethod,
int compressionLevel)
{
- cfp *fp;
+ cfp *fp = NULL;
- if (compressionMethod == COMPRESSION_NONE)
- fp = cfopen(path, mode, compressionMethod, 0);
- else
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ fp = cfopen(path, mode, compressionMethod, 0);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- char *fname;
+ {
+ char *fname;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compressionMethod, compressionLevel);
- free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
+ fatal("not built with zlib support");
#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return fp;
}
@@ -512,7 +1038,7 @@ static cfp *
cfopen_internal(const char *path, int fd, const char *mode,
CompressionMethod compressionMethod, int compressionLevel)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
fp->compressionMethod = compressionMethod;
@@ -563,6 +1089,27 @@ cfopen_internal(const char *path, int fd, const char *mode,
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ LZ4File *lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (fd >= 0)
+ lz4fp->fp = fdopen(fd, mode);
+ else
+ lz4fp->fp = fopen(path, mode);
+ if (lz4fp->fp == NULL)
+ {
+ free_keep_errno(lz4fp);
+ fp = NULL;
+ }
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+ fp->fp = lz4fp;
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -583,8 +1130,8 @@ cfopen(const char *path, const char *mode,
cfp *
cfdopen(int fd, const char *mode,
- CompressionMethod compressionMethod,
- int compressionLevel)
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
}
@@ -620,7 +1167,16 @@ cfread(void *ptr, int size, cfp *fp)
fatal("not built with zlib support");
#endif
break;
-
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_read(fp->fp, ptr, size, false);
+ if (ret != size && !LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s",
+ LZ4File_error(fp->fp));
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
default:
fatal("invalid compression method");
break;
@@ -644,6 +1200,13 @@ cfwrite(const void *ptr, int size, cfp *fp)
ret = gzwrite(fp->fp, ptr, size);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_write(fp->fp, ptr, size);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -679,6 +1242,20 @@ cfgetc(cfp *fp)
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_getc(fp->fp);
+ if (ret == EOF)
+ {
+ if (!LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -698,13 +1275,19 @@ cfgets(cfp *fp, char *buf, int len)
{
case COMPRESSION_NONE:
ret = fgets(buf, len, fp->fp);
-
break;
case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
ret = gzgets(fp->fp, buf, len);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_gets(fp->fp, buf, len);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -739,6 +1322,14 @@ cfclose(cfp *fp)
fp->fp = NULL;
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_close(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -767,6 +1358,13 @@ cfeof(cfp *fp)
ret = gzeof(fp->fp);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_eof(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -780,23 +1378,42 @@ cfeof(cfp *fp)
const char *
get_cfp_error(cfp *fp)
{
- if (fp->compressionMethod == COMPRESSION_GZIP)
+ const char *errmsg = NULL;
+
+ switch(fp->compressionMethod)
{
+ case COMPRESSION_NONE:
+ errmsg = strerror(errno);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ {
+ int errnum;
+ errmsg = gzerror(fp->fp, &errnum);
- if (errnum != Z_ERRNO)
- return errmsg;
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+ }
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ errmsg = LZ4File_error(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
- return strerror(errno);
+ return errmsg;
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -810,5 +1427,3 @@ hasSuffix(const char *filename, const char *suffix)
suffix,
suffixlen) == 0;
}
-
-#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index d9f6c9a087..c443e698f9 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -78,6 +78,7 @@ enum _dumpPreparedQueries
typedef enum _compressionMethod
{
COMPRESSION_GZIP,
+ COMPRESSION_LZ4,
COMPRESSION_NONE,
COMPRESSION_INVALID
} CompressionMethod;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7d96446f1a..e6f727534b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -353,6 +353,7 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
cfp *sav;
@@ -382,17 +383,27 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compressionMethod == COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compressionMethod != COMPRESSION_NONE &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (AH->compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -2019,6 +2030,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2046,30 +2069,21 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
+ return AH->format;
+#endif
+#ifdef HAVE_LIBLZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
return AH->format;
- }
#endif
fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3681,6 +3695,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
WriteInt(AH, AH->compressionLevel);
+ AH->WriteBytePtr(AH, AH->compressionMethod);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3761,14 +3776,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compressionLevel = Z_DEFAULT_COMPRESSION;
- if (AH->compressionLevel != INT_MIN)
+ if (AH->version >= K_VERS_1_15)
+ AH->compressionMethod = AH->ReadBytePtr(AH);
+ else if (AH->compressionLevel != 0)
+ AH->compressionMethod = COMPRESSION_GZIP;
+
#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ {
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
-#else
- AH->compressionMethod = COMPRESSION_GZIP;
+ AH->compressionMethod = COMPRESSION_NONE;
+ AH->compressionLevel = 0;
+ }
#endif
-
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 837e9d73f5..037bfcf913 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,11 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compressionMethod in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index cd91efdd7a..f15a565ac7 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1002,7 +1002,7 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ printf(_(" -Z, --compress=[gzip,lz4,none][:LEVEL] or [LEVEL]\n"
" compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
@@ -1271,11 +1271,13 @@ parse_compression_method(const char *method,
if (pg_strcasecmp(method, "gzip") == 0)
*compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "lz4") == 0)
+ *compressionMethod = COMPRESSION_LZ4;
else if (pg_strcasecmp(method, "none") == 0)
*compressionMethod = COMPRESSION_NONE;
else
{
- pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ pg_log_error("invalid compression method \"%s\" (gzip, lz4, none)", method);
res = false;
}
@@ -1346,10 +1348,10 @@ parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
if (!res)
return res;
- /* one can set level when method is gzip */
- if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ /* one can set level when a compression method is set */
+ if (*compressionMethod == COMPRESSION_NONE && *compressLevel != INT_MIN)
{
- pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is set");
return false;
}
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index f6bccfe55b..e2c3634583 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -123,12 +123,12 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
[ 'pg_dump', '--compress', 'none:1' ],
- qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is set\E/,
'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 49f04c9477..b18f45358e 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -82,11 +82,14 @@ my %pgdump_runs = (
'postgres',
],
# Give coverage for manually compressed blob.toc files during restore.
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-f',
- "$tempdir/compression_gzip_directory_format/blobs.toc",
- ],
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ },
restore_cmd => [
'pg_restore',
"--file=$tempdir/compression_gzip_directory_format.sql",
@@ -102,12 +105,15 @@ my %pgdump_runs = (
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
- # Give coverage for manually compressed blob.toc files during restore.
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-f',
- "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
- ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/toc.dat",
+ ],
+ },
restore_cmd => [
'pg_restore',
'--jobs=3',
@@ -124,12 +130,96 @@ my %pgdump_runs = (
"--file=$tempdir/compression_gzip_plain_format.sql.gz",
'postgres',
],
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-k', '-d',
- "$tempdir/compression_gzip_plain_format.sql.gz",
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-k', '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
+ },
+ compression_lz4_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=custom', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom_format.sql",
+ "$tempdir/compression_lz4_custom_format.dump",
+ ],
+ },
+ compression_lz4_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=directory', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{'LZ4'},
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format/toc.dat",
+ "$tempdir/compression_lz4_directory_format/toc.dat.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_directory_format.sql",
+ "$tempdir/compression_lz4_directory_format",
],
},
+ compression_lz4_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync', '--jobs=2',
+ '--format=directory', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{'LZ4'},
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc",
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_directory_format_parallel.sql",
+ "$tempdir/compression_lz4_directory_format_parallel",
+ ],
+ },
+ # Check that the output is valid lz4
+ compression_lz4_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--no-sync',
+ '--format=plain', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_plain_format.sql.lz4",
+ 'postgres',
+ ],
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{LZ4}, '-d', '-f',
+ "$tempdir/compression_lz4_plain_format.sql.lz4",
+ "$tempdir/compression_lz4_plain_format.sql",
+ ],
+ }
+ },
compression_none_dir_format => {
test_key => 'compression',
dump_cmd => [
@@ -138,10 +228,13 @@ my %pgdump_runs = (
"--file=$tempdir/compression_none_dir_format",
'postgres',
],
- glob_match => {
- no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
- match => "$tempdir/compression_none_dir_format/*.dat",
- match_count => 2, # toc.dat and more
+ compression => {
+ method => 'gzip',
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
},
restore_cmd => [
'pg_restore', '-Fd',
@@ -149,6 +242,26 @@ my %pgdump_runs = (
"$tempdir/compression_none_dir_format",
],
},
+ compression_default_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ "--file=$tempdir/compression_default_dir_format",
+ 'postgres',
+ ],
+ compression => {
+ method => 'gzip',
+ glob_match => {
+ match => "$tempdir/compression_default_dir_format/*.dat.gz",
+ match_count => 1, # data
+ },
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_default_dir_format.sql",
+ "$tempdir/compression_default_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -3953,45 +4066,48 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
my $run_db = 'postgres';
- my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
- my $compress_program = $ENV{GZIP_PROGRAM};
+ my $gzip_program = defined($ENV{GZIP_PROGRAM}) && $ENV{GZIP_PROGRAM} ne '';
+ my $lz4_program = defined($ENV{LZ4}) && $ENV{LZ4} ne '';
+ my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+ my $supports_compression = $supports_gzip || $supports_lz4;
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
- if (defined($pgdump_runs{$run}->{compress_cmd}))
- {
- # Skip compression_cmd tests when compression is not supported,
- # as the result is uncompressed or the utility program does not
- # exist
- next if !$supports_compression || !defined($compress_program)
- || $compress_program eq '';
- command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
- "$run: compression commands");
- }
- if (defined($pgdump_runs{$run}->{glob_match}))
+ if (defined($pgdump_runs{$run}->{'compression'}))
{
# Skip compression_cmd tests when compression is not supported,
# as the result is uncompressed or the utility program does not
# exist
- next if !$supports_compression || !defined($compress_program)
- || $compress_program eq '';
+ next if !$supports_compression;
- my $match = $pgdump_runs{$run}->{glob_match}->{match};
- my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
- $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
- my @glob_matched = glob $match;
+ my ($compression) = $pgdump_runs{$run}->{'compression'};
- cmp_ok(scalar(@glob_matched), '>=', $match_count,
- "Expected at least $match_count file(s) matching $match");
+ next if ${compression}->{method} eq 'gzip' && (!$gzip_program || !$supports_gzip);
+ next if ${compression}->{method} eq 'lz4' && (!$lz4_program || !$supports_lz4);
- if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ if (defined($compression->{cmd}))
{
- my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
- my @glob_matched = glob $no_match;
+ command_ok( \@{ $compression->{cmd} }, "$run: compression commands");
+ }
+
+ if (defined($compression->{glob_match}))
+ {
+ my $match = $compression->{glob_match}->{match};
+ my $match_count = $compression->{glob_match}->{match_count};
+ my @glob_matched = glob $match;
+
+ cmp_ok(scalar(@glob_matched), '>=', $match_count,
+ "Expected at least $match_count file(s) matching $match");
- cmp_ok(scalar(@glob_matched), '==', 0,
- "Expected no file(s) matching $no_match");
+ if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ {
+ my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
+ my @glob_not_matched = glob $no_match;
+
+ cmp_ok(scalar(@glob_not_matched), '==', 0,
+ "Expected no file(s) matching $no_match");
+ }
}
}
--
2.33.0
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
@ 2022-03-25 23:43 ` [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-03-25 23:43 UTC (permalink / raw)
To: Rachel Heaton <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Greg Stark <[email protected]>; Postgres hackers <[email protected]>
------- Original Message -------
On Saturday, March 26th, 2022 at 12:13 AM, Rachel Heaton <[email protected]> wrote:
> On Fri, Mar 25, 2022 at 6:22 AM Justin Pryzby [email protected] wrote:
>
> > On Fri, Mar 25, 2022 at 01:20:47AM -0400, Greg Stark wrote:
> >
> > > It seems development on this has stalled. If there's no further work
> > > happening I guess I'll mark the patch returned with feedback. Feel
> > > free to resubmit it to the next CF when there's progress.
We had some progress yet we didn't want to distract the list with too many
emails. Of course, it seemed stalled to the outside observer, yet I simply
wanted to set the record straight and say that we are actively working on it.
> >
> > Since it's a reasonably large patch (and one that I had myself started before)
> > and it's only been 20some days since (minor) review comments, and since the
> > focus right now is on committing features, and not reviewing new patches, and
> > this patch is new one month ago, and its 0002 not intended for pg15, therefor
> > I'm moving it to the next CF, where I hope to work with its authors to progress
> > it.
Thank you. It is much appreciated. We will sent updates when the next commitfest
starts in July as to not distract from the 15 work. Then, we can take it from there.
>
> Hi Folks,
>
> Here is an updated patchset from Georgios, with minor assistance from myself.
> The comments above should be addressed, but please let us know if
A small amendment to the above statement. This patchset does not include the
refactoring of compress_io suggested by Mr Paquier in the same thread, as it is
missing documentation. An updated version will be sent to include those changes
on the next commitfest.
> there are other things to go over. A functional change in this
> patchset is when `--compress=none` is passed to pg_dump, it will not
> compress for directory type (previously, it would use gzip if
> present). The previous default behavior is retained.
>
> - Rachel
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-03-26 05:57 ` Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-03-26 05:57 UTC (permalink / raw)
To: [email protected]; +Cc: Rachel Heaton <[email protected]>; Justin Pryzby <[email protected]>; Greg Stark <[email protected]>; Postgres hackers <[email protected]>
On Fri, Mar 25, 2022 at 11:43:17PM +0000, [email protected] wrote:
> On Saturday, March 26th, 2022 at 12:13 AM, Rachel Heaton <[email protected]> wrote:
>> Here is an updated patchset from Georgios, with minor assistance from myself.
>> The comments above should be addressed, but please let us know if
>
> A small amendment to the above statement. This patchset does not include the
> refactoring of compress_io suggested by Mr Paquier in the same thread, as it is
> missing documentation. An updated version will be sent to include those changes
> on the next commitfest.
The refactoring using callbacks would make the code much cleaner IMO
in the long term, with zstd waiting in the queue. Now, I see some
pieces of the patch set that could be merged now without waiting for
the development cycle of 16 to begin, as of 0001 to add more tests and
0002.
I have a question about 0002, actually. What has led you to the
conclusion that this code is dead and could be removed?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-03-26 06:14 ` Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-03-26 06:14 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Sat, Mar 26, 2022 at 02:57:50PM +0900, Michael Paquier wrote:
> I have a question about 0002, actually. What has led you to the
> conclusion that this code is dead and could be removed?
See 0001 and the manpage.
+ 'pg_dump: compression is not supported by tar archive format');
When I submitted a patch to support zstd, I spent awhile trying to make
compression work with tar, but it's a significant effort and better done
separately.
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-29 07:27 ` Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-03-29 07:27 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Sat, Mar 26, 2022 at 01:14:41AM -0500, Justin Pryzby wrote:
> See 0001 and the manpage.
>
> + 'pg_dump: compression is not supported by tar archive format');
>
> When I submitted a patch to support zstd, I spent awhile trying to make
> compression work with tar, but it's a significant effort and better done
> separately.
Wow. This stuff is old enough to vote (c3e18804), dead since its
introduction. There is indeed an argument for removing that, it is
not good to keep around that that has never been stressed and/or
used. Upon review, the cleanup done looks correct, as we have never
been able to generate .dat.gz files in for a dump in the tar format.
+ command_fails_like(
+ [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
This addition depending on HAVE_LIBZ is a good thing as a reminder of
any work that could be done in 0002. Now that's waiting for 20 years
so I would not hold my breath on this support. I think that this
could be just applied first, with 0002 on top of it, as a first
improvement.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
Patch 0001 is missing and update of pg_dump's Makefile to pass down
this environment variable to the test scripts, no?
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
[...]
+ $ENV{'GZIP_PROGRAM'},
+ '-k', '-d',
-f and -d are available everywhere I looked at, but is -k/--keep a
portable choice with a gzip command? I don't see this option in
OpenBSD, for one. So this test is going to cause problems on those
buildfarm machines, at least. Couldn't this part be replaced by a
simple --test to check that what has been compressed is in correct
shape? We know that this works, based on our recent experiences with
the other tests.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-03-29 09:46 ` [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-03-29 09:46 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
------- Original Message -------
On Tuesday, March 29th, 2022 at 9:27 AM, Michael Paquier <[email protected]> wrote:
> On Sat, Mar 26, 2022 at 01:14:41AM -0500, Justin Pryzby wrote:
> > See 0001 and the manpage.
> > + 'pg_dump: compression is not supported by tar archive format');
> > When I submitted a patch to support zstd, I spent awhile trying to make
> > compression work with tar, but it's a significant effort and better done
> > separately.
>
> Wow. This stuff is old enough to vote (c3e18804), dead since its
> introduction. There is indeed an argument for removing that, it is
> not good to keep around that that has never been stressed and/or
> used. Upon review, the cleanup done looks correct, as we have never
> been able to generate .dat.gz files in for a dump in the tar format.
Correct. My driving force behind it was to ease up the cleanup/refactoring
work that follows, by eliminating the callers of the GZ*() macros.
> + command_fails_like(
>
> + [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
> This addition depending on HAVE_LIBZ is a good thing as a reminder of
> any work that could be done in 0002. Now that's waiting for 20 years
> so I would not hold my breath on this support. I think that this
> could be just applied first, with 0002 on top of it, as a first
> improvement.
Excellent, thank you.
> + compress_cmd => [
> + $ENV{'GZIP_PROGRAM'},
> Patch 0001 is missing and update of pg_dump's Makefile to pass down
> this environment variable to the test scripts, no?
Agreed. It was not properly moved forward. Fixed.
> + compress_cmd => [
> + $ENV{'GZIP_PROGRAM'},
> + '-f',
> [...]
> + $ENV{'GZIP_PROGRAM'},
> + '-k', '-d',
> -f and -d are available everywhere I looked at, but is -k/--keep a
> portable choice with a gzip command? I don't see this option in
> OpenBSD, for one. So this test is going to cause problems on those
> buildfarm machines, at least. Couldn't this part be replaced by a
> simple --test to check that what has been compressed is in correct
> shape? We know that this works, based on our recent experiences with
> the other tests.
I would argue that the simple '--test' will not do in this case, as the
TAP tests do need a file named <test>.sql to compare the contents with.
This file is generated either directly by pg_dump itself, or by running
pg_restore on pg_dump's output. In the case of compression pg_dump will
generate a <test>.sql.<compression program suffix> which can not be
used in the comparison tests. So the intention of this block, is not to
simply test for validity, but to also decompress pg_dump's output for it
to be able to be used.
I updated the patch to simply remove the '-k' flag.
Please find v3 attached. (only 0001 and 0002 are relevant, 0003 and
0004 are only for reference and are currently under active modification).
Cheers,
//Georgios
Attachments:
[text/x-patch] v3-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (46.7K, ../../yTrMijGhpyxYVVeJuJPM5NandUq_5Qq647cqNuUpVkXSfbYPDj0VLfRK72jB2HWoeDgRIYIbjG7YbBrUuzv8fRF9Y5vALIrM9BGIgQLh8vM=@pm.me/2-v3-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From ef373439ac3c5ec7663ff191b28f744cdc529e86 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Mar 2022 08:31:47 +0000
Subject: [PATCH v3 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_io.{c,h} there are two distinct APIs exposed, the streaming API
and a file API. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
In the later case, the API is using an opaque wrapper around a file stream,
which aquired via fopen() or gzopen() respectively. It would then provide
wrappers around fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(); or
their gz equivallents. However the LZ4F api does not provide this functionality.
So this has been implemented localy.
In order to maintain the API compatibility a new structure LZ4File is
introduced. It is responsible for keeping state and any yet unused generated
content. The later is required when the generated decompressed output, exceeds
the caller's buffer capacity.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression is
still stored in the dump, though admittedly is of no apparent use.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_io.c | 738 ++++++++++++++++++++++++---
src/bin/pg_dump/pg_backup.h | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 71 ++-
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_dump.c | 12 +-
src/bin/pg_dump/t/001_basic.pl | 4 +-
src/bin/pg_dump/t/002_pg_dump.pl | 207 ++++++--
9 files changed, 911 insertions(+), 149 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 992b7312df..68a7c6a3bf 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 2f524b09bf..2864ccabb9 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 630f9e4b18..790f225ec4 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -56,6 +58,14 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBLZ4
+#include "lz4.h"
+#include "lz4frame.h"
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -69,9 +79,9 @@ struct CompressorState
#ifdef HAVE_LIBZ
z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
#endif
+ void *outbuf;
+ size_t outsize;
};
/* Routines that support zlib compressed data I/O */
@@ -85,6 +95,15 @@ static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
#endif
+/* Routines that support LZ4 compressed data I/O */
+#ifdef HAVE_LIBLZ4
+static void InitCompressorLZ4(CompressorState *cs);
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+#endif
+
/* Routines that support uncompressed data I/O */
static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
@@ -103,6 +122,11 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
+#ifndef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ fatal("not built with LZ4 support");
+#endif
+
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
@@ -115,6 +139,10 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
InitCompressorZlib(cs, compressionLevel);
#endif
+#ifdef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ InitCompressorLZ4(cs);
+#endif
return cs;
}
@@ -137,6 +165,13 @@ ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
ReadDataFromArchiveZlib(AH, readF);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ReadDataFromArchiveLZ4(AH, readF);
+#else
+ fatal("not built with lz4 support");
#endif
break;
default:
@@ -159,6 +194,13 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ WriteDataToArchiveLZ4(AH, cs, data, dLen);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -183,6 +225,13 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
EndCompressorZlib(AH, cs);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ EndCompressorLZ4(AH, cs);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -213,20 +262,20 @@ InitCompressorZlib(CompressorState *cs, int level)
zp->opaque = Z_NULL;
/*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
+ * outsize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to append a
* trailing zero byte to the zlib output.
*/
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
+ cs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ cs->outsize = ZLIB_OUT_SIZE;
if (deflateInit(zp, level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
/* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = cs->outbuf;
+ zp->avail_out = cs->outsize;
}
static void
@@ -243,7 +292,7 @@ EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
if (deflateEnd(zp) != Z_OK)
fatal("could not close compression stream: %s", zp->msg);
- free(cs->zlibOut);
+ free(cs->outbuf);
free(cs->zp);
}
@@ -251,7 +300,7 @@ static void
DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
{
z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
+ void *out = cs->outbuf;
int res = Z_OK;
while (cs->zp->avail_in != 0 || flush)
@@ -259,7 +308,7 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
if (res == Z_STREAM_ERROR)
fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
+ if ((flush && (zp->avail_out < cs->outsize))
|| (zp->avail_out == 0)
|| (zp->avail_in != 0)
)
@@ -269,18 +318,18 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
* chunk is the EOF marker in the custom format. This should never
* happen but...
*/
- if (zp->avail_out < cs->zlibOutSize)
+ if (zp->avail_out < cs->outsize)
{
/*
* Any write function should do its own error checking but to
* make sure we do a check here as well...
*/
- size_t len = cs->zlibOutSize - zp->avail_out;
+ size_t len = cs->outsize - zp->avail_out;
- cs->writeF(AH, out, len);
+ cs->writeF(AH, (char *)out, len);
}
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = out;
+ zp->avail_out = cs->outsize;
}
if (res == Z_STREAM_END)
@@ -364,6 +413,71 @@ ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
}
#endif /* HAVE_LIBZ */
+#ifdef HAVE_LIBLZ4
+static void
+InitCompressorLZ4(CompressorState *cs)
+{
+ /* Will be lazy init'd */
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ pg_free(cs->outbuf);
+
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = (4 * 1024) + 1;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen)
+{
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > cs->outsize)
+ {
+ cs->outbuf = pg_realloc(cs->outbuf, requiredsize);
+ cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, cs->outbuf,
+ dLen, cs->outsize);
+
+ if (compressed <= 0)
+ fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, cs->outbuf, compressed);
+}
+#endif /* HAVE_LIBLZ4 */
/*
* Functions for uncompressed output.
@@ -400,9 +514,36 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*----------------------
*/
+#ifdef HAVE_LIBLZ4
/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
+ * State needed for LZ4 (de)compression using the cfp API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+#endif
+
+/*
+ * cfp represents an open stream, wrapping the underlying FILE, gzFile
+ * pointer, or LZ4File pointer. This is opaque to the callers.
*/
struct cfp
{
@@ -410,9 +551,7 @@ struct cfp
void *fp;
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -424,26 +563,380 @@ free_keep_errno(void *p)
errno = save_errno;
}
+#ifdef HAVE_LIBLZ4
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(LZ4File *fs)
+{
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_error(LZ4File *fs)
+{
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File *fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen, &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File *fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ readlen = p - fs->overflowbuf + 1; /* Include the line terminating char */
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read(LZ4File *fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *)readbuf;
+ rend = (char *)readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr
+ * if the eol flag is set, either skip if one already found or fill up to EOL
+ * if present in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size -1 -dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true && (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t)(p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *)ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf, fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int)dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static int
+LZ4File_write(LZ4File *fs, const void *ptr, int size)
+{
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fgetc() and gzgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(LZ4File *fs)
+{
+ unsigned char c;
+
+ if (LZ4File_read(fs, &c, 1, false) != 1)
+ return EOF;
+
+ return c;
+}
+
+/*
+ * fgets() and gzgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(LZ4File *fs, char *buf, int len)
+{
+ size_t dsize;
+
+ dsize = LZ4File_read(fs, buf, len, true);
+ if (dsize < 0)
+ fatal("failed to read from archive %s", LZ4File_error(fs));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return buf;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(LZ4File *fs)
+{
+ FILE *fp;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ fatal("failed to end decompression: %s", LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+#endif
+
/*
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * this will open either "foo" or "foo.gz" or "foo.lz4", trying in that order.
*
* On failure, return NULL with an error code in errno.
+ *
*/
cfp *
cfopen_read(const char *path, const char *mode)
{
- cfp *fp;
+ cfp *fp = NULL;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
+ else if (hasSuffix(path, ".lz4"))
+ fp = cfopen(path, mode, COMPRESSION_LZ4, 0);
else
-#endif
{
fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
@@ -455,8 +948,19 @@ cfopen_read(const char *path, const char *mode)
fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
+#endif
+#ifdef HAVE_LIBLZ4
+ if (fp == NULL)
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, COMPRESSION_LZ4, 0);
+ free_keep_errno(fname);
+ }
#endif
}
+
return fp;
}
@@ -465,9 +969,13 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * When 'compressionMethod' indicates gzip, a gzip compressed stream is opened,
+ * and 'compressionLevel' is used. The ".gz" suffix is automatically added to
+ * 'path' in that case. The same applies when 'compressionMethod' indicates lz4,
+ * but then the ".lz4" suffix is added instead.
+ *
+ * It is the caller's responsibility to verify that the requested
+ * 'compressionMethod' is supported by the build.
*
* On failure, return NULL with an error code in errno.
*/
@@ -476,23 +984,44 @@ cfopen_write(const char *path, const char *mode,
CompressionMethod compressionMethod,
int compressionLevel)
{
- cfp *fp;
+ cfp *fp = NULL;
- if (compressionMethod == COMPRESSION_NONE)
- fp = cfopen(path, mode, compressionMethod, 0);
- else
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ fp = cfopen(path, mode, compressionMethod, 0);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- char *fname;
+ {
+ char *fname;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compressionMethod, compressionLevel);
- free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
+ fatal("not built with zlib support");
#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return fp;
}
@@ -509,7 +1038,7 @@ static cfp *
cfopen_internal(const char *path, int fd, const char *mode,
CompressionMethod compressionMethod, int compressionLevel)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
fp->compressionMethod = compressionMethod;
@@ -560,6 +1089,27 @@ cfopen_internal(const char *path, int fd, const char *mode,
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ LZ4File *lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (fd >= 0)
+ lz4fp->fp = fdopen(fd, mode);
+ else
+ lz4fp->fp = fopen(path, mode);
+ if (lz4fp->fp == NULL)
+ {
+ free_keep_errno(lz4fp);
+ fp = NULL;
+ }
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+ fp->fp = lz4fp;
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -580,8 +1130,8 @@ cfopen(const char *path, const char *mode,
cfp *
cfdopen(int fd, const char *mode,
- CompressionMethod compressionMethod,
- int compressionLevel)
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
}
@@ -617,7 +1167,16 @@ cfread(void *ptr, int size, cfp *fp)
fatal("not built with zlib support");
#endif
break;
-
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_read(fp->fp, ptr, size, false);
+ if (ret != size && !LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s",
+ LZ4File_error(fp->fp));
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
default:
fatal("invalid compression method");
break;
@@ -641,6 +1200,13 @@ cfwrite(const void *ptr, int size, cfp *fp)
ret = gzwrite(fp->fp, ptr, size);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_write(fp->fp, ptr, size);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -676,6 +1242,20 @@ cfgetc(cfp *fp)
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_getc(fp->fp);
+ if (ret == EOF)
+ {
+ if (!LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -695,13 +1275,19 @@ cfgets(cfp *fp, char *buf, int len)
{
case COMPRESSION_NONE:
ret = fgets(buf, len, fp->fp);
-
break;
case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
ret = gzgets(fp->fp, buf, len);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_gets(fp->fp, buf, len);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -736,6 +1322,14 @@ cfclose(cfp *fp)
fp->fp = NULL;
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_close(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -764,6 +1358,13 @@ cfeof(cfp *fp)
ret = gzeof(fp->fp);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_eof(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -777,23 +1378,42 @@ cfeof(cfp *fp)
const char *
get_cfp_error(cfp *fp)
{
- if (fp->compressionMethod == COMPRESSION_GZIP)
+ const char *errmsg = NULL;
+
+ switch(fp->compressionMethod)
{
+ case COMPRESSION_NONE:
+ errmsg = strerror(errno);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ {
+ int errnum;
+ errmsg = gzerror(fp->fp, &errnum);
- if (errnum != Z_ERRNO)
- return errmsg;
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+ }
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ errmsg = LZ4File_error(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
- return strerror(errno);
+ return errmsg;
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -807,5 +1427,3 @@ hasSuffix(const char *filename, const char *suffix)
suffix,
suffixlen) == 0;
}
-
-#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index d9f6c9a087..c443e698f9 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -78,6 +78,7 @@ enum _dumpPreparedQueries
typedef enum _compressionMethod
{
COMPRESSION_GZIP,
+ COMPRESSION_LZ4,
COMPRESSION_NONE,
COMPRESSION_INVALID
} CompressionMethod;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7d96446f1a..e6f727534b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -353,6 +353,7 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
cfp *sav;
@@ -382,17 +383,27 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compressionMethod == COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compressionMethod != COMPRESSION_NONE &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (AH->compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -2019,6 +2030,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2046,30 +2069,21 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
+ return AH->format;
+#endif
+#ifdef HAVE_LIBLZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
return AH->format;
- }
#endif
fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3681,6 +3695,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
WriteInt(AH, AH->compressionLevel);
+ AH->WriteBytePtr(AH, AH->compressionMethod);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3761,14 +3776,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compressionLevel = Z_DEFAULT_COMPRESSION;
- if (AH->compressionLevel != INT_MIN)
+ if (AH->version >= K_VERS_1_15)
+ AH->compressionMethod = AH->ReadBytePtr(AH);
+ else if (AH->compressionLevel != 0)
+ AH->compressionMethod = COMPRESSION_GZIP;
+
#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ {
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
-#else
- AH->compressionMethod = COMPRESSION_GZIP;
+ AH->compressionMethod = COMPRESSION_NONE;
+ AH->compressionLevel = 0;
+ }
#endif
-
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 837e9d73f5..037bfcf913 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,11 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compressionMethod in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 97ac17ebff..5351c71d2b 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1002,7 +1002,7 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ printf(_(" -Z, --compress=[gzip,lz4,none][:LEVEL] or [LEVEL]\n"
" compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
@@ -1271,11 +1271,13 @@ parse_compression_method(const char *method,
if (pg_strcasecmp(method, "gzip") == 0)
*compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "lz4") == 0)
+ *compressionMethod = COMPRESSION_LZ4;
else if (pg_strcasecmp(method, "none") == 0)
*compressionMethod = COMPRESSION_NONE;
else
{
- pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ pg_log_error("invalid compression method \"%s\" (gzip, lz4, none)", method);
res = false;
}
@@ -1346,10 +1348,10 @@ parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
if (!res)
return res;
- /* one can set level when method is gzip */
- if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ /* one can set level when a compression method is set */
+ if (*compressionMethod == COMPRESSION_NONE && *compressLevel != INT_MIN)
{
- pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is set");
return false;
}
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 4713d0c8d4..e8d792e08a 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,12 +122,12 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
[ 'pg_dump', '--compress', 'none:1' ],
- qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is set\E/,
'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 152d5564eb..032aa94760 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -81,11 +81,14 @@ my %pgdump_runs = (
'postgres',
],
# Give coverage for manually compressed blob.toc files during restore.
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-f',
- "$tempdir/compression_gzip_directory_format/blobs.toc",
- ],
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ },
restore_cmd => [
'pg_restore',
"--file=$tempdir/compression_gzip_directory_format.sql",
@@ -101,12 +104,15 @@ my %pgdump_runs = (
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
- # Give coverage for manually compressed blob.toc files during restore.
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-f',
- "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
- ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/toc.dat",
+ ],
+ },
restore_cmd => [
'pg_restore',
'--jobs=3',
@@ -123,12 +129,97 @@ my %pgdump_runs = (
"--file=$tempdir/compression_gzip_plain_format.sql.gz",
'postgres',
],
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-d',
- "$tempdir/compression_gzip_plain_format.sql.gz",
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
+ },
+ compression_lz4_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=custom', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom_format.sql",
+ "$tempdir/compression_lz4_custom_format.dump",
+ ],
+ },
+ compression_lz4_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=directory', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{'LZ4'},
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format/toc.dat",
+ "$tempdir/compression_lz4_directory_format/toc.dat.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_directory_format.sql",
+ "$tempdir/compression_lz4_directory_format",
+>>>>>>> f6385a1dce (Add LZ4 compression in pg_{dump|restore})
],
},
+ compression_lz4_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync', '--jobs=2',
+ '--format=directory', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{'LZ4'},
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc",
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_directory_format_parallel.sql",
+ "$tempdir/compression_lz4_directory_format_parallel",
+ ],
+ },
+ # Check that the output is valid lz4
+ compression_lz4_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--no-sync',
+ '--format=plain', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_plain_format.sql.lz4",
+ 'postgres',
+ ],
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{LZ4}, '-d', '-f',
+ "$tempdir/compression_lz4_plain_format.sql.lz4",
+ "$tempdir/compression_lz4_plain_format.sql",
+ ],
+ }
+ },
compression_none_dir_format => {
test_key => 'compression',
dump_cmd => [
@@ -137,10 +228,13 @@ my %pgdump_runs = (
"--file=$tempdir/compression_none_dir_format",
'postgres',
],
- glob_match => {
- no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
- match => "$tempdir/compression_none_dir_format/*.dat",
- match_count => 2, # toc.dat and more
+ compression => {
+ method => 'gzip',
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
},
restore_cmd => [
'pg_restore', '-Fd',
@@ -148,6 +242,26 @@ my %pgdump_runs = (
"$tempdir/compression_none_dir_format",
],
},
+ compression_default_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ "--file=$tempdir/compression_default_dir_format",
+ 'postgres',
+ ],
+ compression => {
+ method => 'gzip',
+ glob_match => {
+ match => "$tempdir/compression_default_dir_format/*.dat.gz",
+ match_count => 1, # data
+ },
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_default_dir_format.sql",
+ "$tempdir/compression_default_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -4044,45 +4158,48 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
my $run_db = 'postgres';
- my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
- my $compress_program = $ENV{GZIP_PROGRAM};
+ my $gzip_program = defined($ENV{GZIP_PROGRAM}) && $ENV{GZIP_PROGRAM} ne '';
+ my $lz4_program = defined($ENV{LZ4}) && $ENV{LZ4} ne '';
+ my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+ my $supports_compression = $supports_gzip || $supports_lz4;
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
- if (defined($pgdump_runs{$run}->{compress_cmd}))
- {
- # Skip compression_cmd tests when compression is not supported,
- # as the result is uncompressed or the utility program does not
- # exist
- next if !$supports_compression || !defined($compress_program)
- || $compress_program eq '';
- command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
- "$run: compression commands");
- }
- if (defined($pgdump_runs{$run}->{glob_match}))
+ if (defined($pgdump_runs{$run}->{'compression'}))
{
# Skip compression_cmd tests when compression is not supported,
# as the result is uncompressed or the utility program does not
# exist
- next if !$supports_compression || !defined($compress_program)
- || $compress_program eq '';
+ next if !$supports_compression;
- my $match = $pgdump_runs{$run}->{glob_match}->{match};
- my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
- $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
- my @glob_matched = glob $match;
+ my ($compression) = $pgdump_runs{$run}->{'compression'};
- cmp_ok(scalar(@glob_matched), '>=', $match_count,
- "Expected at least $match_count file(s) matching $match");
+ next if ${compression}->{method} eq 'gzip' && (!$gzip_program || !$supports_gzip);
+ next if ${compression}->{method} eq 'lz4' && (!$lz4_program || !$supports_lz4);
- if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ if (defined($compression->{cmd}))
{
- my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
- my @glob_matched = glob $no_match;
+ command_ok( \@{ $compression->{cmd} }, "$run: compression commands");
+ }
+
+ if (defined($compression->{glob_match}))
+ {
+ my $match = $compression->{glob_match}->{match};
+ my $match_count = $compression->{glob_match}->{match_count};
+ my @glob_matched = glob $match;
+
+ cmp_ok(scalar(@glob_matched), '>=', $match_count,
+ "Expected at least $match_count file(s) matching $match");
- cmp_ok(scalar(@glob_matched), '==', 0,
- "Expected no file(s) matching $no_match");
+ if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ {
+ my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
+ my @glob_not_matched = glob $no_match;
+
+ cmp_ok(scalar(@glob_not_matched), '==', 0,
+ "Expected no file(s) matching $no_match");
+ }
}
}
--
2.32.0
[text/x-patch] v3-0003-Prepare-pg_dump-for-additional-compression-method.patch (52.9K, ../../yTrMijGhpyxYVVeJuJPM5NandUq_5Qq647cqNuUpVkXSfbYPDj0VLfRK72jB2HWoeDgRIYIbjG7YbBrUuzv8fRF9Y5vALIrM9BGIgQLh8vM=@pm.me/3-v3-0003-Prepare-pg_dump-for-additional-compression-method.patch)
download | inline diff:
From d70f2bf62900f0e3feae97fb9ec4043c1aca35e9 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Mar 2022 08:30:15 +0000
Subject: [PATCH v3 3/4] Prepare pg_dump for additional compression methods
This commmit does the heavy lifting required for additional compression methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
Furthermore, compression was chosen based on the value of the level passed
as an argument during the invocation of pg_dump or some hardcoded defaults. This
does not scale for more than one compression methods. Now the method used for
compression can be explicitly requested during command invocation, or set during
hardcoded defaults. Then it is stored in the relevant structs and passed in the
relevant functions, along side compression level which has lost it's special
meaning. The method for compression is not yet stored in the actual archive.
This is done in the next commit which does introduce a new method.
The previously named CompressionAlgorithm enum is changed for
CompressionMethod so that it matches better similar variables found through out
the code base.
In a fashion similar to the binary for pg_basebackup, the method for compression
is passed using the already existing -Z/--compress parameter of pg_dump. The
legacy format and behaviour is maintained. Additionally, the user can explicitly
pass a requested method and optionaly the level to be used after a semicolon,
e.g. --compress=gzip:6
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 416 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 32 +-
src/bin/pg_dump/pg_backup.h | 14 +-
src/bin/pg_dump/pg_backup_archiver.c | 171 +++++------
src/bin/pg_dump/pg_backup_archiver.h | 46 +--
src/bin/pg_dump/pg_backup_custom.c | 11 +-
src/bin/pg_dump/pg_backup_directory.c | 12 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 155 ++++++++--
src/bin/pg_dump/t/001_basic.pl | 14 +-
src/bin/pg_dump/t/002_pg_dump.pl | 50 +++-
src/tools/pgindent/typedefs.list | 2 +-
13 files changed, 608 insertions(+), 357 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2f0042fd96..992b7312df 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 9077fdb74d..630f9e4b18 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ CompressionMethod compressionMethod;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compressionMethod = compressionMethod;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (compressionMethod == COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compressionLevel);
#endif
return cs;
@@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
+ int compressionLevel, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +152,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compressionMethod)
{
- case COMPR_ALG_LIBZ:
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compressionMethod)
+ {
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ CompressionMethod compressionMethod;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -455,18 +441,18 @@ cfopen_read(const char *path, const char *mode)
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
#endif
@@ -486,19 +472,21 @@ cfopen_read(const char *path, const char *mode)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compressionMethod == COMPRESSION_NONE)
+ fp = cfopen(path, mode, compressionMethod, 0);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
free_keep_errno(fname);
#else
fatal("not built with zlib support");
@@ -509,60 +497,94 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compressionMethod' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ CompressionMethod compressionMethod, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compressionMethod = compressionMethod;
+
+ switch (compressionMethod)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(path, -1, mode, compressionMethod, compressionLevel);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +594,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +656,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- fatal("could not read from input file: %s", strerror(errno));
- else
- fatal("could not read from input file: end of file");
- }
- }
- else
+ case COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile)fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +689,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compressionMethod)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compressionMethod == COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..b8b366616c 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -17,16 +17,17 @@
#include "pg_backup_archiver.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#else
+/* this is just the redefinition of a libz constant */
+#define Z_DEFAULT_COMPRESSION (-1)
+#endif
+
/* Initial buffer sizes used in zlib compression. */
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +47,12 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ CompressionMethod compressionMethod,
+ int compressionLevel,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +61,16 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+extern cfp *cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index fcc5f6bd05..d9f6c9a087 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -75,6 +75,13 @@ enum _dumpPreparedQueries
NUM_PREP_QUERIES /* must be last */
};
+typedef enum _compressionMethod
+{
+ COMPRESSION_GZIP,
+ COMPRESSION_NONE,
+ COMPRESSION_INVALID
+} CompressionMethod;
+
/* Parameters needed by ConnectDatabase; same for dump and restore */
typedef struct _connParams
{
@@ -143,7 +150,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ CompressionMethod compressionMethod;
+ int compressionLevel;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +311,9 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compression,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index d41a99d6ea..7d96446f1a 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,9 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +94,11 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +237,15 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt,
+ compressionMethod, compressionLevel,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +255,8 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, COMPRESSION_NONE, 0, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
@@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compressionMethod == COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +459,9 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename,
+ ropt->compressionMethod, ropt->compressionLevel);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -740,7 +741,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -970,6 +971,7 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compressionMethod = COMPRESSION_NONE;
return opts;
}
@@ -1117,13 +1119,13 @@ PrintTOCSummary(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, COMPRESSION_NONE, 0);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1132,7 +1134,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compressionLevel);
switch (AH->format)
{
@@ -1486,60 +1488,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod, int compressionLevel)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compressionMethod, compressionLevel);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compressionMethod, compressionLevel);
if (!AH->OF)
{
@@ -1550,33 +1527,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *)AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1700,22 +1668,16 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
-
- else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're
+ * connected then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ else
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2200,7 +2162,9 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2251,14 +2215,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compressionMethod = compressionMethod;
+ AH->compressionLevel = compressionLevel;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, COMPRESSION_NONE, 0);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2266,7 +2230,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compressionMethod != COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3716,7 +3680,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compressionLevel);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3790,18 +3754,21 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compressionLevel = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compressionLevel = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compressionLevel = Z_DEFAULT_COMPRESSION;
+ if (AH->compressionLevel != INT_MIN)
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+#else
+ AH->compressionMethod = COMPRESSION_GZIP;
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 540d4f6a83..837e9d73f5 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,17 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ CompressionMethod compressionMethod; /* Requested method for compression */
+ int compressionLevel; /*---------
+ * Requested level of compression for method.
+ * Possible values for compression:
+ * INT_MIN when no compression method is
+ * requested.
+ * -1 Z_DEFAULT_COMPRESSION for gzip
+ * compression.
+ * 1-9 levels for gzip compression.
+ *---------
+ */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..7f38ea9cd5 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -377,7 +379,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +570,8 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compressionMethod, AH->compressionLevel,
+ _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7f4e340dea..0e60b447de 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod,
+ AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W,
+ COMPRESSION_NONE, 0);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -644,7 +647,7 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", COMPRESSION_NONE, 0);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +665,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod, AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 620e609055..3e60afd39d 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -801,7 +802,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -890,7 +890,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
sfx = ".gz";
else
sfx = "";
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 535b160165..97ac17ebff 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -55,6 +55,7 @@
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
#include "common/connect.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -163,6 +164,9 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression_option(const char *opt,
+ CompressionMethod *compressionMethod,
+ int *compressLevel);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -336,8 +340,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ int compressLevel = INT_MIN;
+ CompressionMethod compressionMethod = COMPRESSION_INVALID;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -557,9 +562,9 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression_option(optarg, &compressionMethod,
+ &compressLevel))
exit_nicely(1);
break;
@@ -689,23 +694,21 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /* Set default compressionMethod unless one already set by the user */
+ if (compressionMethod == COMPRESSION_INVALID)
{
+ compressionMethod = COMPRESSION_NONE;
+
#ifdef HAVE_LIBZ
+ /* Custom and directory formats are compressed by default (zlib) */
if (archiveFormat == archCustom || archiveFormat == archDirectory)
+ {
+ compressionMethod = COMPRESSION_GZIP;
compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ }
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -718,8 +721,9 @@ main(int argc, char **argv)
fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat,
+ compressionMethod, compressLevel,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -950,10 +954,8 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compressionLevel = compressLevel;
+ ropt->compressionMethod = compressionMethod;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -1000,7 +1002,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ " compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1260,6 +1263,116 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+static bool
+parse_compression_method(const char *method,
+ CompressionMethod *compressionMethod)
+{
+ bool res = true;
+
+ if (pg_strcasecmp(method, "gzip") == 0)
+ *compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "none") == 0)
+ *compressionMethod = COMPRESSION_NONE;
+ else
+ {
+ pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ res = false;
+ }
+
+ return res;
+}
+
+/*
+ * Interprets a compression option of the format 'method[:LEVEL]' of legacy just
+ * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level
+ * are returned in *compressionMethod and *compressionLevel. In case of error,
+ * the function returns false and then the values of *compression{Method,Level}
+ * are not to be trusted.
+ */
+static bool
+parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
+ int *compressLevel)
+{
+ char *method;
+ const char *sep;
+ int methodlen;
+ bool supports_compression = true;
+ bool res = true;
+
+ /* find the separator if exists */
+ sep = strchr(opt, ':');
+
+ /*
+ * If there is no separator, then it is either a legacy format, or only the
+ * method has been passed.
+ */
+ if (!sep)
+ {
+ if (strspn(opt, "-0123456789") == strlen(opt))
+ {
+ res = option_parse_int(opt, "-Z/--compress", 0, 9, compressLevel);
+ *compressionMethod = (*compressLevel > 0) ? COMPRESSION_GZIP :
+ COMPRESSION_NONE;
+ }
+ else
+ res = parse_compression_method(opt, compressionMethod);
+ }
+ else
+ {
+ /* otherwise, it should be method:LEVEL */
+ methodlen = sep - opt + 1;
+ method = pg_malloc0(methodlen);
+ snprintf(method, methodlen, "%.*s", methodlen - 1, opt);
+
+ res = parse_compression_method(method, compressionMethod);
+ if (res)
+ {
+ sep++;
+ if (*sep == '\0')
+ {
+ pg_log_error("no level defined for compression \"%s\"", method);
+ pg_free(method);
+ res = false;
+ }
+ else
+ {
+ res = option_parse_int(sep, "-Z/--compress [LEVEL]", 1, 9,
+ compressLevel);
+ }
+ }
+ }
+
+ /* if there is an error, there is no need to check further */
+ if (!res)
+ return res;
+
+ /* one can set level when method is gzip */
+ if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ {
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ return false;
+ }
+
+ /* verify that the requested compression is supported */
+#ifndef HAVE_LIBZ
+ if (*compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (*compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ *compressionMethod = COMPRESSION_NONE;
+ *compressLevel = INT_MIN;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 9c1238181a..4713d0c8d4 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -120,6 +120,16 @@ command_fails_like(
qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/,
'pg_restore: cannot specify both --single-transaction and multiple jobs');
+command_fails_like(
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ 'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
+
command_fails_like(
[ 'pg_dump', '-Z', '-1' ],
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
@@ -128,14 +138,14 @@ command_fails_like(
if (check_pg_config("#define HAVE_LIBZ 1"))
{
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
}
else
{
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: requested compression not available in this installation -- archive will be uncompressed');
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index b9f6dccec4..152d5564eb 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -76,7 +76,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_directory_format",
'postgres',
],
@@ -97,7 +97,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=6',
+ '--format=directory', '--compress=gzip:6',
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
@@ -129,6 +129,25 @@ my %pgdump_runs = (
"$tempdir/compression_gzip_plain_format.sql.gz",
],
},
+ compression_none_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ '--compress=none',
+ "--file=$tempdir/compression_none_dir_format",
+ 'postgres',
+ ],
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_none_dir_format.sql",
+ "$tempdir/compression_none_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -228,7 +247,7 @@ my %pgdump_runs = (
defaults_dir_format => {
test_key => 'defaults',
dump_cmd => [
- 'pg_dump', '-Fd',
+ 'pg_dump', '-Fd',
"--file=$tempdir/defaults_dir_format", 'postgres',
],
restore_cmd => [
@@ -4041,6 +4060,31 @@ foreach my $run (sort keys %pgdump_runs)
command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
"$run: compression commands");
}
+ if (defined($pgdump_runs{$run}->{glob_match}))
+ {
+ # Skip compression_cmd tests when compression is not supported,
+ # as the result is uncompressed or the utility program does not
+ # exist
+ next if !$supports_compression || !defined($compress_program)
+ || $compress_program eq '';
+
+ my $match = $pgdump_runs{$run}->{glob_match}->{match};
+ my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
+ $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
+ my @glob_matched = glob $match;
+
+ cmp_ok(scalar(@glob_matched), '>=', $match_count,
+ "Expected at least $match_count file(s) matching $match");
+
+ if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ {
+ my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
+ my @glob_matched = glob $no_match;
+
+ cmp_ok(scalar(@glob_matched), '==', 0,
+ "Expected no file(s) matching $no_match");
+ }
+ }
if ($pgdump_runs{$run}->{restore_cmd})
{
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 182b233b4c..00c6ad9516 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -412,7 +412,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
-CompressionAlgorithm
+CompressionMethod
CompressorState
ComputeXidHorizonsResult
ConditionVariable
--
2.32.0
[text/x-patch] v3-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch (5.9K, ../../yTrMijGhpyxYVVeJuJPM5NandUq_5Qq647cqNuUpVkXSfbYPDj0VLfRK72jB2HWoeDgRIYIbjG7YbBrUuzv8fRF9Y5vALIrM9BGIgQLh8vM=@pm.me/4-v3-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch)
download | inline diff:
From 65232a1ed8fdfb57775bef856a907965dfcda14d Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Mar 2022 08:29:15 +0000
Subject: [PATCH v3 1/4] Extend compression coverage for pg_dump, pg_restore
---
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/t/001_basic.pl | 15 ++++++
src/bin/pg_dump/t/002_pg_dump.pl | 92 ++++++++++++++++++++++++++++++++
3 files changed, 109 insertions(+)
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 302f7e02d6..2f524b09bf 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
+export GZIP_PROGRAM=$(GZIP)
+
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index e0bf3eb032..9c1238181a 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -125,6 +125,21 @@ command_fails_like(
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
'pg_dump: -Z/--compress must be in range');
+if (check_pg_config("#define HAVE_LIBZ 1"))
+{
+ command_fails_like(
+ [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
+ 'pg_dump: compression is not supported by tar archive format');
+}
+else
+{
+ command_fails_like(
+ [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
+ 'pg_dump: warning: requested compression not available in this installation -- archive will be uncompressed');
+}
+
command_fails_like(
[ 'pg_dump', '--extra-float-digits', '-16' ],
qr/\Qpg_dump: error: --extra-float-digits must be in range\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index af5d6fa5a3..b9f6dccec4 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -54,6 +54,81 @@ my %pgdump_runs = (
"$tempdir/binary_upgrade.dump",
],
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=custom', '--compress=1',
+ "--file=$tempdir/compression_gzip_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_custom_format.sql",
+ "$tempdir/compression_gzip_custom_format.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=directory', '--compress=1',
+ "--file=$tempdir/compression_gzip_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_directory_format.sql",
+ "$tempdir/compression_gzip_directory_format",
+ ],
+ },
+
+ compression_gzip_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=6',
+ "--file=$tempdir/compression_gzip_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ '--jobs=3',
+ "--file=$tempdir/compression_gzip_directory_format_parallel.sql",
+ "$tempdir/compression_gzip_directory_format_parallel",
+ ],
+ },
+
+ # Check that the output is valid gzip
+ compression_gzip_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '-Z1',
+ "--file=$tempdir/compression_gzip_plain_format.sql.gz",
+ 'postgres',
+ ],
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -424,6 +499,7 @@ my %full_runs = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3098,6 +3174,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_test_table => 1,
@@ -3171,6 +3248,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3947,9 +4025,23 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
my $run_db = 'postgres';
+ my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
+ my $compress_program = $ENV{GZIP_PROGRAM};
+
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
+ if (defined($pgdump_runs{$run}->{compress_cmd}))
+ {
+ # Skip compression_cmd tests when compression is not supported,
+ # as the result is uncompressed or the utility program does not
+ # exist
+ next if !$supports_compression || !defined($compress_program)
+ || $compress_program eq '';
+ command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
+ "$run: compression commands");
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.32.0
[text/x-patch] v3-0002-Remove-unsupported-bitrot-compression-from-pg_dum.patch (4.1K, ../../yTrMijGhpyxYVVeJuJPM5NandUq_5Qq647cqNuUpVkXSfbYPDj0VLfRK72jB2HWoeDgRIYIbjG7YbBrUuzv8fRF9Y5vALIrM9BGIgQLh8vM=@pm.me/5-v3-0002-Remove-unsupported-bitrot-compression-from-pg_dum.patch)
download | inline diff:
From 2842f3f09d209319abf02ad2a60efd71399e75b5 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Mar 2022 08:29:25 +0000
Subject: [PATCH v3 2/4] Remove unsupported/bitrot compression from pg_dump tar
---
src/bin/pg_dump/pg_backup_tar.c | 82 ++++-----------------------------
1 file changed, 9 insertions(+), 73 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index ccfbe346be..620e609055 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -65,11 +65,6 @@ static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
typedef struct
{
-#ifdef HAVE_LIBZ
- gzFile zFH;
-#else
- FILE *zFH;
-#endif
FILE *nFH;
FILE *tarFH;
FILE *tmpFH;
@@ -248,14 +243,7 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
ctx->filename = pg_strdup(fn);
}
else
@@ -320,10 +308,6 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
lclContext *ctx = (lclContext *) AH->formatData;
TAR_MEMBER *tm;
-#ifdef HAVE_LIBZ
- char fmode[14];
-#endif
-
if (mode == 'r')
{
tm = _tarPositionTo(AH, filename);
@@ -344,16 +328,10 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
-#ifdef HAVE_LIBZ
-
if (AH->compression == 0)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
- /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
-#else
- tm->nFH = ctx->tarFH;
-#endif
}
else
{
@@ -405,21 +383,10 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
-#ifdef HAVE_LIBZ
-
- if (AH->compression != 0)
- {
- sprintf(fmode, "wb%d", AH->compression);
- tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
- if (tm->zFH == NULL)
- fatal("could not open temporary file");
- }
- else
+ if (AH->compression == 0)
tm->nFH = tm->tmpFH;
-#else
-
- tm->nFH = tm->tmpFH;
-#endif
+ else
+ fatal("compression is not supported by tar archive format");
tm->AH = AH;
tm->targetFile = pg_strdup(filename);
@@ -434,15 +401,8 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- /*
- * Close the GZ file since we dup'd. This will flush the buffers.
- */
if (AH->compression != 0)
- {
- errno = 0; /* in case gzclose() doesn't set it */
- if (GZCLOSE(th->zFH) != 0)
- fatal("could not close tar member: %m");
- }
+ fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
_tarAddFile(AH, th); /* This will close the temp file */
@@ -456,7 +416,6 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
free(th->targetFile);
th->nFH = NULL;
- th->zFH = NULL;
}
#ifdef __NOT_USED__
@@ -542,29 +501,9 @@ _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
}
else if (th)
{
- if (th->zFH)
- {
- res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
- if (res != len && !GZEOF(th->zFH))
- {
-#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(th->zFH, &errnum);
-
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
-#else
- fatal("could not read from input file: %s",
- strerror(errno));
-#endif
- }
- }
- else
- {
- res = fread(&((char *) buf)[used], 1, len, th->nFH);
- if (res != len && !feof(th->nFH))
- READ_ERROR_EXIT(th->nFH);
- }
+ res = fread(&((char *) buf)[used], 1, len, th->nFH);
+ if (res != len && !feof(th->nFH))
+ READ_ERROR_EXIT(th->nFH);
}
}
@@ -596,10 +535,7 @@ tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
{
size_t res;
- if (th->zFH != NULL)
- res = GZWRITE(buf, 1, len, th->zFH);
- else
- res = fwrite(buf, 1, len, th->nFH);
+ res = fwrite(buf, 1, len, th->nFH);
th->pos += res;
return res;
--
2.32.0
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-03-30 05:54 ` Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-03-30 05:54 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Tue, Mar 29, 2022 at 09:46:27AM +0000, [email protected] wrote:
> On Tuesday, March 29th, 2022 at 9:27 AM, Michael Paquier <[email protected]> wrote:
>> On Sat, Mar 26, 2022 at 01:14:41AM -0500, Justin Pryzby wrote:
>> Wow. This stuff is old enough to vote (c3e18804), dead since its
>> introduction. There is indeed an argument for removing that, it is
>> not good to keep around that that has never been stressed and/or
>> used. Upon review, the cleanup done looks correct, as we have never
>> been able to generate .dat.gz files in for a dump in the tar format.
>
> Correct. My driving force behind it was to ease up the cleanup/refactoring
> work that follows, by eliminating the callers of the GZ*() macros.
Makes sense to me.
>> + command_fails_like(
>>
>> + [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
>> This addition depending on HAVE_LIBZ is a good thing as a reminder of
>> any work that could be done in 0002. Now that's waiting for 20 years
>> so I would not hold my breath on this support. I think that this
>> could be just applied first, with 0002 on top of it, as a first
>> improvement.
>
> Excellent, thank you.
I have applied the test for --compress and --format=tar, separating it
from the rest.
While moving on with 0002, I have noticed the following in
_StartBlob():
if (AH->compression != 0)
sfx = ".gz";
else
sfx = "";
Shouldn't this bit also be simplified, adding a fatal() like the other
code paths, for safety?
>> + compress_cmd => [
>> + $ENV{'GZIP_PROGRAM'},
>> + '-f',
>> [...]
>> + $ENV{'GZIP_PROGRAM'},
>> + '-k', '-d',
>> -f and -d are available everywhere I looked at, but is -k/--keep a
>> portable choice with a gzip command? I don't see this option in
>> OpenBSD, for one. So this test is going to cause problems on those
>> buildfarm machines, at least. Couldn't this part be replaced by a
>> simple --test to check that what has been compressed is in correct
>> shape? We know that this works, based on our recent experiences with
>> the other tests.
>
> I would argue that the simple '--test' will not do in this case, as the
> TAP tests do need a file named <test>.sql to compare the contents with.
> This file is generated either directly by pg_dump itself, or by running
> pg_restore on pg_dump's output. In the case of compression pg_dump will
> generate a <test>.sql.<compression program suffix> which can not be
> used in the comparison tests. So the intention of this block, is not to
> simply test for validity, but to also decompress pg_dump's output for it
> to be able to be used.
Ahh, I see, thanks. I would add a comment about that in the area of
compression_gzip_plain_format.
+ my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
This part could be moved within the if block a couple of lines down.
+ my $compress_program = $ENV{GZIP_PROGRAM};
It seems to me that it is enough to rely on {compress_cmd}, hence
there should be no need for $compress_program, no?
It seems to me that we should have a description for compress_cmd at
the top of 002_pg_dump.pl (close to "Definition of the pg_dump runs to
make"). There is an order dependency with restore_cmd.
> I updated the patch to simply remove the '-k' flag.
Okay.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-03-30 15:32 ` [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-03-30 15:32 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
------- Original Message -------
On Wednesday, March 30th, 2022 at 7:54 AM, Michael Paquier <[email protected]> wrote:
> On Tue, Mar 29, 2022 at 09:46:27AM +0000, [email protected] wrote:
> > On Tuesday, March 29th, 2022 at 9:27 AM, Michael Paquier [email protected] wrote:
> > > On Sat, Mar 26, 2022 at 01:14:41AM -0500, Justin Pryzby wrote:
> > > + command_fails_like(
> > > + [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
> > > This addition depending on HAVE_LIBZ is a good thing as a reminder of
> > > any work that could be done in 0002. Now that's waiting for 20 years
> > > so I would not hold my breath on this support. I think that this
> > > could be just applied first, with 0002 on top of it, as a first
> > > improvement.
> >
> > Excellent, thank you.
>
> I have applied the test for --compress and --format=tar, separating it
> from the rest.
Thank you.
> While moving on with 0002, I have noticed the following in
>
> _StartBlob():
> if (AH->compression != 0)
> sfx = ".gz";
> else
> sfx = "";
>
> Shouldn't this bit also be simplified, adding a fatal() like the other
> code paths, for safety?
Agreed. Fixed.
> > > + compress_cmd => [
> > > + $ENV{'GZIP_PROGRAM'},
> > > + '-f',
> > > [...]
> > > + $ENV{'GZIP_PROGRAM'},
> > > + '-k', '-d',
> > > -f and -d are available everywhere I looked at, but is -k/--keep a
> > > portable choice with a gzip command? I don't see this option in
> > > OpenBSD, for one. So this test is going to cause problems on those
> > > buildfarm machines, at least. Couldn't this part be replaced by a
> > > simple --test to check that what has been compressed is in correct
> > > shape? We know that this works, based on our recent experiences with
> > > the other tests.
> >
> > I would argue that the simple '--test' will not do in this case, as the
> > TAP tests do need a file named <test>.sql to compare the contents with.
> > This file is generated either directly by pg_dump itself, or by running
> > pg_restore on pg_dump's output. In the case of compression pg_dump will
> > generate a <test>.sql.<compression program suffix> which can not be
> > used in the comparison tests. So the intention of this block, is not to
> > simply test for validity, but to also decompress pg_dump's output for it
> > to be able to be used.
>
> Ahh, I see, thanks. I would add a comment about that in the area of
> compression_gzip_plain_format.
Agreed. Comment added.
> + my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
>
> This part could be moved within the if block a couple of lines down.
I moved it instead out of the for loop above to not have to call it on
each iteration.
> + my $compress_program = $ENV{GZIP_PROGRAM};
>
> It seems to me that it is enough to rely on {compress_cmd}, hence
> there should be no need for $compress_program, no?
Maybe not. We don't want to the tests to fail if the utility is not
installed. That becomes even more evident as more methods are added.
However I realized that the presence of the environmental variable does
not guarrantee that the utility is actually installed. In the attached,
the existance of the utility is based on the return value of system_log().
> It seems to me that we should have a description for compress_cmd at
> the top of 002_pg_dump.pl (close to "Definition of the pg_dump runs to
> make"). There is an order dependency with restore_cmd.
Agreed. Comment added.
Cheers,
//Georgios
Attachments:
[text/x-patch] v4-0002-Remove-unsupported-bitrot-compression-from-pg_dum.patch (4.7K, ../../bZeDkb5raMHiji1UIjvXjVIwc5qCHcdAb5h3CXXp85D8Mjwrm4kIGPh8UVbc2sJlfKgx2KsDHSrF35JAr-kFW5GJOx8YQlLKLv5vmAYLEjQ=@pm.me/2-v4-0002-Remove-unsupported-bitrot-compression-from-pg_dum.patch)
download | inline diff:
From 0768f208a9f8462d4b1ad8210eb1a977aad15b41 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 30 Mar 2022 15:00:02 +0000
Subject: [PATCH v4 2/4] Remove unsupported/bitrot compression from pg_dump tar
---
src/bin/pg_dump/pg_backup_tar.c | 89 ++++-----------------------------
1 file changed, 11 insertions(+), 78 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index ccfbe346be..2491a091b9 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -65,11 +65,6 @@ static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
typedef struct
{
-#ifdef HAVE_LIBZ
- gzFile zFH;
-#else
- FILE *zFH;
-#endif
FILE *nFH;
FILE *tarFH;
FILE *tmpFH;
@@ -248,14 +243,7 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
ctx->filename = pg_strdup(fn);
}
else
@@ -320,10 +308,6 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
lclContext *ctx = (lclContext *) AH->formatData;
TAR_MEMBER *tm;
-#ifdef HAVE_LIBZ
- char fmode[14];
-#endif
-
if (mode == 'r')
{
tm = _tarPositionTo(AH, filename);
@@ -344,16 +328,10 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
-#ifdef HAVE_LIBZ
-
if (AH->compression == 0)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
- /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
-#else
- tm->nFH = ctx->tarFH;
-#endif
}
else
{
@@ -405,21 +383,10 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
-#ifdef HAVE_LIBZ
-
- if (AH->compression != 0)
- {
- sprintf(fmode, "wb%d", AH->compression);
- tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
- if (tm->zFH == NULL)
- fatal("could not open temporary file");
- }
- else
+ if (AH->compression == 0)
tm->nFH = tm->tmpFH;
-#else
-
- tm->nFH = tm->tmpFH;
-#endif
+ else
+ fatal("compression is not supported by tar archive format");
tm->AH = AH;
tm->targetFile = pg_strdup(filename);
@@ -434,15 +401,8 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- /*
- * Close the GZ file since we dup'd. This will flush the buffers.
- */
if (AH->compression != 0)
- {
- errno = 0; /* in case gzclose() doesn't set it */
- if (GZCLOSE(th->zFH) != 0)
- fatal("could not close tar member: %m");
- }
+ fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
_tarAddFile(AH, th); /* This will close the temp file */
@@ -456,7 +416,6 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
free(th->targetFile);
th->nFH = NULL;
- th->zFH = NULL;
}
#ifdef __NOT_USED__
@@ -542,29 +501,9 @@ _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
}
else if (th)
{
- if (th->zFH)
- {
- res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
- if (res != len && !GZEOF(th->zFH))
- {
-#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(th->zFH, &errnum);
-
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
-#else
- fatal("could not read from input file: %s",
- strerror(errno));
-#endif
- }
- }
- else
- {
- res = fread(&((char *) buf)[used], 1, len, th->nFH);
- if (res != len && !feof(th->nFH))
- READ_ERROR_EXIT(th->nFH);
- }
+ res = fread(&((char *) buf)[used], 1, len, th->nFH);
+ if (res != len && !feof(th->nFH))
+ READ_ERROR_EXIT(th->nFH);
}
}
@@ -596,10 +535,7 @@ tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
{
size_t res;
- if (th->zFH != NULL)
- res = GZWRITE(buf, 1, len, th->zFH);
- else
- res = fwrite(buf, 1, len, th->nFH);
+ res = fwrite(buf, 1, len, th->nFH);
th->pos += res;
return res;
@@ -949,17 +885,14 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
+ fatal("compression is not supported by tar archive format");
- sprintf(fname, "blob_%u.dat%s", oid, sfx);
+ sprintf(fname, "blob_%u.dat", oid);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
--
2.32.0
[text/x-patch] v4-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch (5.7K, ../../bZeDkb5raMHiji1UIjvXjVIwc5qCHcdAb5h3CXXp85D8Mjwrm4kIGPh8UVbc2sJlfKgx2KsDHSrF35JAr-kFW5GJOx8YQlLKLv5vmAYLEjQ=@pm.me/3-v4-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch)
download | inline diff:
From 4e33597455c8fa420bca5bd87013922478996017 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 30 Mar 2022 14:59:10 +0000
Subject: [PATCH v4 1/4] Extend compression coverage for pg_dump, pg_restore
---
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/t/002_pg_dump.pl | 100 +++++++++++++++++++++++++++++++
2 files changed, 102 insertions(+)
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 302f7e02d6..2f524b09bf 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
+export GZIP_PROGRAM=$(GZIP)
+
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index af5d6fa5a3..91c3b978c4 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -26,6 +26,13 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# specified and is pulled from $PGPORT, which is set by the
# PostgreSQL::Test::Cluster system.
#
+# compress_cmd is the utility command for (de)compression, if any.
+# Note that this should generally be used on pg_dump's output
+# either to generate a text file to run the through the tests, or
+# to test pg_restore's ability to parse manually compressed files
+# that otherwise pg_dump does not compress on it's own
+# (e.g. *.toc).
+#
# restore_cmd is the pg_restore command to run, if any. Note
# that this should generally be used when the pg_dump goes to
# a non-text file and that the restore can then be used to
@@ -54,6 +61,82 @@ my %pgdump_runs = (
"$tempdir/binary_upgrade.dump",
],
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=custom', '--compress=1',
+ "--file=$tempdir/compression_gzip_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_custom_format.sql",
+ "$tempdir/compression_gzip_custom_format.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=directory', '--compress=1',
+ "--file=$tempdir/compression_gzip_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_directory_format.sql",
+ "$tempdir/compression_gzip_directory_format",
+ ],
+ },
+
+ compression_gzip_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=6',
+ "--file=$tempdir/compression_gzip_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ '--jobs=3',
+ "--file=$tempdir/compression_gzip_directory_format_parallel.sql",
+ "$tempdir/compression_gzip_directory_format_parallel",
+ ],
+ },
+
+ # Check that the output is valid gzip
+ compression_gzip_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '-Z1',
+ "--file=$tempdir/compression_gzip_plain_format.sql.gz",
+ 'postgres',
+ ],
+ # Decompress the generated file to run through the tests
+ compress_cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -424,6 +507,7 @@ my %full_runs = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3098,6 +3182,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_test_table => 1,
@@ -3171,6 +3256,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3941,6 +4027,9 @@ command_fails_like(
#########################################
# Run all runs
+my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
+my $compress_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
+ '>', '/dev/null') == 0);
foreach my $run (sort keys %pgdump_runs)
{
@@ -3950,6 +4039,17 @@ foreach my $run (sort keys %pgdump_runs)
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
+ if (defined($pgdump_runs{$run}->{compress_cmd}))
+ {
+ # Skip compression_cmd tests when compression is not supported,
+ # as the result is uncompressed or the utility program does not
+ # exist
+ next if !$supports_compression || !$compress_program_exists;
+
+ command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
+ "$run: compression commands");
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.32.0
[text/x-patch] v4-0003-Prepare-pg_dump-for-additional-compression-method.patch (52.8K, ../../bZeDkb5raMHiji1UIjvXjVIwc5qCHcdAb5h3CXXp85D8Mjwrm4kIGPh8UVbc2sJlfKgx2KsDHSrF35JAr-kFW5GJOx8YQlLKLv5vmAYLEjQ=@pm.me/4-v4-0003-Prepare-pg_dump-for-additional-compression-method.patch)
download | inline diff:
From 7d2edf176b1ba4c70253dbeec0e878b9ed515349 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 30 Mar 2022 15:03:53 +0000
Subject: [PATCH v4 3/4] Prepare pg_dump for additional compression methods
This commmit does the heavy lifting required for additional compression methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
Furthermore, compression was chosen based on the value of the level passed
as an argument during the invocation of pg_dump or some hardcoded defaults. This
does not scale for more than one compression methods. Now the method used for
compression can be explicitly requested during command invocation, or set during
hardcoded defaults. Then it is stored in the relevant structs and passed in the
relevant functions, along side compression level which has lost it's special
meaning. The method for compression is not yet stored in the actual archive.
This is done in the next commit which does introduce a new method.
The previously named CompressionAlgorithm enum is changed for
CompressionMethod so that it matches better similar variables found through out
the code base.
In a fashion similar to the binary for pg_basebackup, the method for compression
is passed using the already existing -Z/--compress parameter of pg_dump. The
legacy format and behaviour is maintained. Additionally, the user can explicitly
pass a requested method and optionaly the level to be used after a semicolon,
e.g. --compress=gzip:6
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 416 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 32 +-
src/bin/pg_dump/pg_backup.h | 14 +-
src/bin/pg_dump/pg_backup_archiver.c | 171 +++++------
src/bin/pg_dump/pg_backup_archiver.h | 46 +--
src/bin/pg_dump/pg_backup_custom.c | 11 +-
src/bin/pg_dump/pg_backup_directory.c | 12 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 155 ++++++++--
src/bin/pg_dump/t/001_basic.pl | 14 +-
src/bin/pg_dump/t/002_pg_dump.pl | 45 ++-
src/tools/pgindent/typedefs.list | 2 +-
13 files changed, 603 insertions(+), 357 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2f0042fd96..992b7312df 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 9077fdb74d..630f9e4b18 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ CompressionMethod compressionMethod;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compressionMethod = compressionMethod;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (compressionMethod == COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compressionLevel);
#endif
return cs;
@@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
+ int compressionLevel, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +152,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compressionMethod)
{
- case COMPR_ALG_LIBZ:
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compressionMethod)
+ {
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ CompressionMethod compressionMethod;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -455,18 +441,18 @@ cfopen_read(const char *path, const char *mode)
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
#endif
@@ -486,19 +472,21 @@ cfopen_read(const char *path, const char *mode)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compressionMethod == COMPRESSION_NONE)
+ fp = cfopen(path, mode, compressionMethod, 0);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
free_keep_errno(fname);
#else
fatal("not built with zlib support");
@@ -509,60 +497,94 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compressionMethod' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ CompressionMethod compressionMethod, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compressionMethod = compressionMethod;
+
+ switch (compressionMethod)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(path, -1, mode, compressionMethod, compressionLevel);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +594,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +656,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- fatal("could not read from input file: %s", strerror(errno));
- else
- fatal("could not read from input file: end of file");
- }
- }
- else
+ case COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile)fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +689,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compressionMethod)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compressionMethod == COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..b8b366616c 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -17,16 +17,17 @@
#include "pg_backup_archiver.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#else
+/* this is just the redefinition of a libz constant */
+#define Z_DEFAULT_COMPRESSION (-1)
+#endif
+
/* Initial buffer sizes used in zlib compression. */
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +47,12 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ CompressionMethod compressionMethod,
+ int compressionLevel,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +61,16 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+extern cfp *cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index fcc5f6bd05..7645d3285a 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -75,6 +75,13 @@ enum _dumpPreparedQueries
NUM_PREP_QUERIES /* must be last */
};
+typedef enum _compressionMethod
+{
+ COMPRESSION_INVALID,
+ COMPRESSION_NONE,
+ COMPRESSION_GZIP
+} CompressionMethod;
+
/* Parameters needed by ConnectDatabase; same for dump and restore */
typedef struct _connParams
{
@@ -143,7 +150,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ CompressionMethod compressionMethod;
+ int compressionLevel;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +311,9 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compression,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index d41a99d6ea..7d96446f1a 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,9 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +94,11 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +237,15 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt,
+ compressionMethod, compressionLevel,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +255,8 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, COMPRESSION_NONE, 0, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
@@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compressionMethod == COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +459,9 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename,
+ ropt->compressionMethod, ropt->compressionLevel);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -740,7 +741,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -970,6 +971,7 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compressionMethod = COMPRESSION_NONE;
return opts;
}
@@ -1117,13 +1119,13 @@ PrintTOCSummary(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, COMPRESSION_NONE, 0);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1132,7 +1134,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compressionLevel);
switch (AH->format)
{
@@ -1486,60 +1488,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod, int compressionLevel)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compressionMethod, compressionLevel);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compressionMethod, compressionLevel);
if (!AH->OF)
{
@@ -1550,33 +1527,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *)AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1700,22 +1668,16 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
-
- else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're
+ * connected then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ else
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2200,7 +2162,9 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2251,14 +2215,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compressionMethod = compressionMethod;
+ AH->compressionLevel = compressionLevel;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, COMPRESSION_NONE, 0);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2266,7 +2230,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compressionMethod != COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3716,7 +3680,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compressionLevel);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3790,18 +3754,21 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compressionLevel = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compressionLevel = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compressionLevel = Z_DEFAULT_COMPRESSION;
+ if (AH->compressionLevel != INT_MIN)
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+#else
+ AH->compressionMethod = COMPRESSION_GZIP;
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 540d4f6a83..837e9d73f5 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,17 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ CompressionMethod compressionMethod; /* Requested method for compression */
+ int compressionLevel; /*---------
+ * Requested level of compression for method.
+ * Possible values for compression:
+ * INT_MIN when no compression method is
+ * requested.
+ * -1 Z_DEFAULT_COMPRESSION for gzip
+ * compression.
+ * 1-9 levels for gzip compression.
+ *---------
+ */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..7f38ea9cd5 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -377,7 +379,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +570,8 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compressionMethod, AH->compressionLevel,
+ _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7f4e340dea..0e60b447de 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod,
+ AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W,
+ COMPRESSION_NONE, 0);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -644,7 +647,7 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", COMPRESSION_NONE, 0);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +665,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod, AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 2491a091b9..b25b641caa 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -801,7 +802,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -889,7 +889,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 535b160165..97ac17ebff 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -55,6 +55,7 @@
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
#include "common/connect.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -163,6 +164,9 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression_option(const char *opt,
+ CompressionMethod *compressionMethod,
+ int *compressLevel);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -336,8 +340,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ int compressLevel = INT_MIN;
+ CompressionMethod compressionMethod = COMPRESSION_INVALID;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -557,9 +562,9 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression_option(optarg, &compressionMethod,
+ &compressLevel))
exit_nicely(1);
break;
@@ -689,23 +694,21 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /* Set default compressionMethod unless one already set by the user */
+ if (compressionMethod == COMPRESSION_INVALID)
{
+ compressionMethod = COMPRESSION_NONE;
+
#ifdef HAVE_LIBZ
+ /* Custom and directory formats are compressed by default (zlib) */
if (archiveFormat == archCustom || archiveFormat == archDirectory)
+ {
+ compressionMethod = COMPRESSION_GZIP;
compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ }
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -718,8 +721,9 @@ main(int argc, char **argv)
fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat,
+ compressionMethod, compressLevel,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -950,10 +954,8 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compressionLevel = compressLevel;
+ ropt->compressionMethod = compressionMethod;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -1000,7 +1002,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ " compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1260,6 +1263,116 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+static bool
+parse_compression_method(const char *method,
+ CompressionMethod *compressionMethod)
+{
+ bool res = true;
+
+ if (pg_strcasecmp(method, "gzip") == 0)
+ *compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "none") == 0)
+ *compressionMethod = COMPRESSION_NONE;
+ else
+ {
+ pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ res = false;
+ }
+
+ return res;
+}
+
+/*
+ * Interprets a compression option of the format 'method[:LEVEL]' of legacy just
+ * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level
+ * are returned in *compressionMethod and *compressionLevel. In case of error,
+ * the function returns false and then the values of *compression{Method,Level}
+ * are not to be trusted.
+ */
+static bool
+parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
+ int *compressLevel)
+{
+ char *method;
+ const char *sep;
+ int methodlen;
+ bool supports_compression = true;
+ bool res = true;
+
+ /* find the separator if exists */
+ sep = strchr(opt, ':');
+
+ /*
+ * If there is no separator, then it is either a legacy format, or only the
+ * method has been passed.
+ */
+ if (!sep)
+ {
+ if (strspn(opt, "-0123456789") == strlen(opt))
+ {
+ res = option_parse_int(opt, "-Z/--compress", 0, 9, compressLevel);
+ *compressionMethod = (*compressLevel > 0) ? COMPRESSION_GZIP :
+ COMPRESSION_NONE;
+ }
+ else
+ res = parse_compression_method(opt, compressionMethod);
+ }
+ else
+ {
+ /* otherwise, it should be method:LEVEL */
+ methodlen = sep - opt + 1;
+ method = pg_malloc0(methodlen);
+ snprintf(method, methodlen, "%.*s", methodlen - 1, opt);
+
+ res = parse_compression_method(method, compressionMethod);
+ if (res)
+ {
+ sep++;
+ if (*sep == '\0')
+ {
+ pg_log_error("no level defined for compression \"%s\"", method);
+ pg_free(method);
+ res = false;
+ }
+ else
+ {
+ res = option_parse_int(sep, "-Z/--compress [LEVEL]", 1, 9,
+ compressLevel);
+ }
+ }
+ }
+
+ /* if there is an error, there is no need to check further */
+ if (!res)
+ return res;
+
+ /* one can set level when method is gzip */
+ if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ {
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ return false;
+ }
+
+ /* verify that the requested compression is supported */
+#ifndef HAVE_LIBZ
+ if (*compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (*compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ *compressionMethod = COMPRESSION_NONE;
+ *compressLevel = INT_MIN;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 65e6c01fed..d7a52ac1a8 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -120,6 +120,16 @@ command_fails_like(
qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/,
'pg_restore: cannot specify both --single-transaction and multiple jobs');
+command_fails_like(
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ 'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
+
command_fails_like(
[ 'pg_dump', '-Z', '-1' ],
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
@@ -128,7 +138,7 @@ command_fails_like(
if (check_pg_config("#define HAVE_LIBZ 1"))
{
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
}
@@ -136,7 +146,7 @@ else
{
# --jobs > 1 forces an error with tar format.
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar', '-j3' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: compression not available in this installation');
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 91c3b978c4..3a3dad2cc7 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -83,7 +83,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_directory_format",
'postgres',
],
@@ -104,7 +104,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=6',
+ '--format=directory', '--compress=gzip:6',
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
@@ -137,6 +137,25 @@ my %pgdump_runs = (
"$tempdir/compression_gzip_plain_format.sql.gz",
],
},
+ compression_none_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ '--compress=none',
+ "--file=$tempdir/compression_none_dir_format",
+ 'postgres',
+ ],
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_none_dir_format.sql",
+ "$tempdir/compression_none_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -236,7 +255,7 @@ my %pgdump_runs = (
defaults_dir_format => {
test_key => 'defaults',
dump_cmd => [
- 'pg_dump', '-Fd',
+ 'pg_dump', '-Fd',
"--file=$tempdir/defaults_dir_format", 'postgres',
],
restore_cmd => [
@@ -4048,6 +4067,26 @@ foreach my $run (sort keys %pgdump_runs)
command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
"$run: compression commands");
+
+ if (defined($pgdump_runs{$run}->{glob_match}))
+ {
+ my $match = $pgdump_runs{$run}->{glob_match}->{match};
+ my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
+ $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
+ my @glob_matched = glob $match;
+
+ cmp_ok(scalar(@glob_matched), '>=', $match_count,
+ "Expected at least $match_count file(s) matching $match");
+
+ if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
+ {
+ my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
+ my @glob_matched = glob $no_match;
+
+ cmp_ok(scalar(@glob_matched), '==', 0,
+ "Expected no file(s) matching $no_match");
+ }
+ }
}
if ($pgdump_runs{$run}->{restore_cmd})
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 72fafb795b..5ac8c156a9 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -412,7 +412,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
-CompressionAlgorithm
+CompressionMethod
CompressorState
ComputeXidHorizonsResult
ConditionVariable
--
2.32.0
[text/x-patch] v4-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (46.3K, ../../bZeDkb5raMHiji1UIjvXjVIwc5qCHcdAb5h3CXXp85D8Mjwrm4kIGPh8UVbc2sJlfKgx2KsDHSrF35JAr-kFW5GJOx8YQlLKLv5vmAYLEjQ=@pm.me/5-v4-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From d3104c7cc1fc86fa36f7b67f645c5b72898de48c Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 30 Mar 2022 15:11:01 +0000
Subject: [PATCH v4 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_io.{c,h} there are two distinct APIs exposed, the streaming API
and a file API. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
In the later case, the API is using an opaque wrapper around a file stream,
which aquired via fopen() or gzopen() respectively. It would then provide
wrappers around fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(); or
their gz equivallents. However the LZ4F api does not provide this functionality.
So this has been implemented localy.
In order to maintain the API compatibility a new structure LZ4File is
introduced. It is responsible for keeping state and any yet unused generated
content. The later is required when the generated decompressed output, exceeds
the caller's buffer capacity.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression is
still stored in the dump, though admittedly is of no apparent use.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_io.c | 738 ++++++++++++++++++++++++---
src/bin/pg_dump/pg_backup.h | 3 +-
src/bin/pg_dump/pg_backup_archiver.c | 71 ++-
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_dump.c | 12 +-
src/bin/pg_dump/t/001_basic.pl | 4 +-
src/bin/pg_dump/t/002_pg_dump.pl | 185 +++++--
9 files changed, 903 insertions(+), 137 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 992b7312df..68a7c6a3bf 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 2f524b09bf..2864ccabb9 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 630f9e4b18..790f225ec4 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -56,6 +58,14 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBLZ4
+#include "lz4.h"
+#include "lz4frame.h"
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -69,9 +79,9 @@ struct CompressorState
#ifdef HAVE_LIBZ
z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
#endif
+ void *outbuf;
+ size_t outsize;
};
/* Routines that support zlib compressed data I/O */
@@ -85,6 +95,15 @@ static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
#endif
+/* Routines that support LZ4 compressed data I/O */
+#ifdef HAVE_LIBLZ4
+static void InitCompressorLZ4(CompressorState *cs);
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+#endif
+
/* Routines that support uncompressed data I/O */
static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
@@ -103,6 +122,11 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
+#ifndef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ fatal("not built with LZ4 support");
+#endif
+
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
@@ -115,6 +139,10 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
InitCompressorZlib(cs, compressionLevel);
#endif
+#ifdef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ InitCompressorLZ4(cs);
+#endif
return cs;
}
@@ -137,6 +165,13 @@ ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
ReadDataFromArchiveZlib(AH, readF);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ReadDataFromArchiveLZ4(AH, readF);
+#else
+ fatal("not built with lz4 support");
#endif
break;
default:
@@ -159,6 +194,13 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ WriteDataToArchiveLZ4(AH, cs, data, dLen);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -183,6 +225,13 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
EndCompressorZlib(AH, cs);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ EndCompressorLZ4(AH, cs);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -213,20 +262,20 @@ InitCompressorZlib(CompressorState *cs, int level)
zp->opaque = Z_NULL;
/*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
+ * outsize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to append a
* trailing zero byte to the zlib output.
*/
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
+ cs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ cs->outsize = ZLIB_OUT_SIZE;
if (deflateInit(zp, level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
/* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = cs->outbuf;
+ zp->avail_out = cs->outsize;
}
static void
@@ -243,7 +292,7 @@ EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
if (deflateEnd(zp) != Z_OK)
fatal("could not close compression stream: %s", zp->msg);
- free(cs->zlibOut);
+ free(cs->outbuf);
free(cs->zp);
}
@@ -251,7 +300,7 @@ static void
DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
{
z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
+ void *out = cs->outbuf;
int res = Z_OK;
while (cs->zp->avail_in != 0 || flush)
@@ -259,7 +308,7 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
if (res == Z_STREAM_ERROR)
fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
+ if ((flush && (zp->avail_out < cs->outsize))
|| (zp->avail_out == 0)
|| (zp->avail_in != 0)
)
@@ -269,18 +318,18 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
* chunk is the EOF marker in the custom format. This should never
* happen but...
*/
- if (zp->avail_out < cs->zlibOutSize)
+ if (zp->avail_out < cs->outsize)
{
/*
* Any write function should do its own error checking but to
* make sure we do a check here as well...
*/
- size_t len = cs->zlibOutSize - zp->avail_out;
+ size_t len = cs->outsize - zp->avail_out;
- cs->writeF(AH, out, len);
+ cs->writeF(AH, (char *)out, len);
}
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = out;
+ zp->avail_out = cs->outsize;
}
if (res == Z_STREAM_END)
@@ -364,6 +413,71 @@ ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
}
#endif /* HAVE_LIBZ */
+#ifdef HAVE_LIBLZ4
+static void
+InitCompressorLZ4(CompressorState *cs)
+{
+ /* Will be lazy init'd */
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ pg_free(cs->outbuf);
+
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = (4 * 1024) + 1;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen)
+{
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > cs->outsize)
+ {
+ cs->outbuf = pg_realloc(cs->outbuf, requiredsize);
+ cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, cs->outbuf,
+ dLen, cs->outsize);
+
+ if (compressed <= 0)
+ fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, cs->outbuf, compressed);
+}
+#endif /* HAVE_LIBLZ4 */
/*
* Functions for uncompressed output.
@@ -400,9 +514,36 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*----------------------
*/
+#ifdef HAVE_LIBLZ4
/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
+ * State needed for LZ4 (de)compression using the cfp API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+#endif
+
+/*
+ * cfp represents an open stream, wrapping the underlying FILE, gzFile
+ * pointer, or LZ4File pointer. This is opaque to the callers.
*/
struct cfp
{
@@ -410,9 +551,7 @@ struct cfp
void *fp;
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -424,26 +563,380 @@ free_keep_errno(void *p)
errno = save_errno;
}
+#ifdef HAVE_LIBLZ4
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(LZ4File *fs)
+{
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_error(LZ4File *fs)
+{
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File *fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen, &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File *fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ readlen = p - fs->overflowbuf + 1; /* Include the line terminating char */
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read(LZ4File *fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *)readbuf;
+ rend = (char *)readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr
+ * if the eol flag is set, either skip if one already found or fill up to EOL
+ * if present in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size -1 -dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true && (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t)(p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *)ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf, fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int)dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static int
+LZ4File_write(LZ4File *fs, const void *ptr, int size)
+{
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fgetc() and gzgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(LZ4File *fs)
+{
+ unsigned char c;
+
+ if (LZ4File_read(fs, &c, 1, false) != 1)
+ return EOF;
+
+ return c;
+}
+
+/*
+ * fgets() and gzgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(LZ4File *fs, char *buf, int len)
+{
+ size_t dsize;
+
+ dsize = LZ4File_read(fs, buf, len, true);
+ if (dsize < 0)
+ fatal("failed to read from archive %s", LZ4File_error(fs));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return buf;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(LZ4File *fs)
+{
+ FILE *fp;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ fatal("failed to end decompression: %s", LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+#endif
+
/*
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * this will open either "foo" or "foo.gz" or "foo.lz4", trying in that order.
*
* On failure, return NULL with an error code in errno.
+ *
*/
cfp *
cfopen_read(const char *path, const char *mode)
{
- cfp *fp;
+ cfp *fp = NULL;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
+ else if (hasSuffix(path, ".lz4"))
+ fp = cfopen(path, mode, COMPRESSION_LZ4, 0);
else
-#endif
{
fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
@@ -455,8 +948,19 @@ cfopen_read(const char *path, const char *mode)
fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
+#endif
+#ifdef HAVE_LIBLZ4
+ if (fp == NULL)
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, COMPRESSION_LZ4, 0);
+ free_keep_errno(fname);
+ }
#endif
}
+
return fp;
}
@@ -465,9 +969,13 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * When 'compressionMethod' indicates gzip, a gzip compressed stream is opened,
+ * and 'compressionLevel' is used. The ".gz" suffix is automatically added to
+ * 'path' in that case. The same applies when 'compressionMethod' indicates lz4,
+ * but then the ".lz4" suffix is added instead.
+ *
+ * It is the caller's responsibility to verify that the requested
+ * 'compressionMethod' is supported by the build.
*
* On failure, return NULL with an error code in errno.
*/
@@ -476,23 +984,44 @@ cfopen_write(const char *path, const char *mode,
CompressionMethod compressionMethod,
int compressionLevel)
{
- cfp *fp;
+ cfp *fp = NULL;
- if (compressionMethod == COMPRESSION_NONE)
- fp = cfopen(path, mode, compressionMethod, 0);
- else
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ fp = cfopen(path, mode, compressionMethod, 0);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- char *fname;
+ {
+ char *fname;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compressionMethod, compressionLevel);
- free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
+ fatal("not built with zlib support");
#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return fp;
}
@@ -509,7 +1038,7 @@ static cfp *
cfopen_internal(const char *path, int fd, const char *mode,
CompressionMethod compressionMethod, int compressionLevel)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
fp->compressionMethod = compressionMethod;
@@ -560,6 +1089,27 @@ cfopen_internal(const char *path, int fd, const char *mode,
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ LZ4File *lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (fd >= 0)
+ lz4fp->fp = fdopen(fd, mode);
+ else
+ lz4fp->fp = fopen(path, mode);
+ if (lz4fp->fp == NULL)
+ {
+ free_keep_errno(lz4fp);
+ fp = NULL;
+ }
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+ fp->fp = lz4fp;
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -580,8 +1130,8 @@ cfopen(const char *path, const char *mode,
cfp *
cfdopen(int fd, const char *mode,
- CompressionMethod compressionMethod,
- int compressionLevel)
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
}
@@ -617,7 +1167,16 @@ cfread(void *ptr, int size, cfp *fp)
fatal("not built with zlib support");
#endif
break;
-
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_read(fp->fp, ptr, size, false);
+ if (ret != size && !LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s",
+ LZ4File_error(fp->fp));
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
default:
fatal("invalid compression method");
break;
@@ -641,6 +1200,13 @@ cfwrite(const void *ptr, int size, cfp *fp)
ret = gzwrite(fp->fp, ptr, size);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_write(fp->fp, ptr, size);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -676,6 +1242,20 @@ cfgetc(cfp *fp)
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_getc(fp->fp);
+ if (ret == EOF)
+ {
+ if (!LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -695,13 +1275,19 @@ cfgets(cfp *fp, char *buf, int len)
{
case COMPRESSION_NONE:
ret = fgets(buf, len, fp->fp);
-
break;
case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
ret = gzgets(fp->fp, buf, len);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_gets(fp->fp, buf, len);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -736,6 +1322,14 @@ cfclose(cfp *fp)
fp->fp = NULL;
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_close(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -764,6 +1358,13 @@ cfeof(cfp *fp)
ret = gzeof(fp->fp);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_eof(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -777,23 +1378,42 @@ cfeof(cfp *fp)
const char *
get_cfp_error(cfp *fp)
{
- if (fp->compressionMethod == COMPRESSION_GZIP)
+ const char *errmsg = NULL;
+
+ switch(fp->compressionMethod)
{
+ case COMPRESSION_NONE:
+ errmsg = strerror(errno);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ {
+ int errnum;
+ errmsg = gzerror(fp->fp, &errnum);
- if (errnum != Z_ERRNO)
- return errmsg;
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+ }
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ errmsg = LZ4File_error(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
- return strerror(errno);
+ return errmsg;
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -807,5 +1427,3 @@ hasSuffix(const char *filename, const char *suffix)
suffix,
suffixlen) == 0;
}
-
-#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 7645d3285a..10393f3fc4 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -78,8 +78,9 @@ enum _dumpPreparedQueries
typedef enum _compressionMethod
{
COMPRESSION_INVALID,
+ COMPRESSION_GZIP,
COMPRESSION_NONE,
- COMPRESSION_GZIP
+ COMPRESSION_LZ4
} CompressionMethod;
/* Parameters needed by ConnectDatabase; same for dump and restore */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7d96446f1a..e6f727534b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -353,6 +353,7 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
cfp *sav;
@@ -382,17 +383,27 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compressionMethod == COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compressionMethod != COMPRESSION_NONE &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (AH->compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -2019,6 +2030,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2046,30 +2069,21 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
+ return AH->format;
+#endif
+#ifdef HAVE_LIBLZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
return AH->format;
- }
#endif
fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3681,6 +3695,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
WriteInt(AH, AH->compressionLevel);
+ AH->WriteBytePtr(AH, AH->compressionMethod);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3761,14 +3776,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compressionLevel = Z_DEFAULT_COMPRESSION;
- if (AH->compressionLevel != INT_MIN)
+ if (AH->version >= K_VERS_1_15)
+ AH->compressionMethod = AH->ReadBytePtr(AH);
+ else if (AH->compressionLevel != 0)
+ AH->compressionMethod = COMPRESSION_GZIP;
+
#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ {
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
-#else
- AH->compressionMethod = COMPRESSION_GZIP;
+ AH->compressionMethod = COMPRESSION_NONE;
+ AH->compressionLevel = 0;
+ }
#endif
-
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 837e9d73f5..037bfcf913 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,11 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compressionMethod in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 97ac17ebff..5351c71d2b 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1002,7 +1002,7 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ printf(_(" -Z, --compress=[gzip,lz4,none][:LEVEL] or [LEVEL]\n"
" compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
@@ -1271,11 +1271,13 @@ parse_compression_method(const char *method,
if (pg_strcasecmp(method, "gzip") == 0)
*compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "lz4") == 0)
+ *compressionMethod = COMPRESSION_LZ4;
else if (pg_strcasecmp(method, "none") == 0)
*compressionMethod = COMPRESSION_NONE;
else
{
- pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ pg_log_error("invalid compression method \"%s\" (gzip, lz4, none)", method);
res = false;
}
@@ -1346,10 +1348,10 @@ parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
if (!res)
return res;
- /* one can set level when method is gzip */
- if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ /* one can set level when a compression method is set */
+ if (*compressionMethod == COMPRESSION_NONE && *compressLevel != INT_MIN)
{
- pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is set");
return false;
}
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index d7a52ac1a8..0077288388 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,12 +122,12 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
[ 'pg_dump', '--compress', 'none:1' ],
- qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is set\E/,
'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 3a3dad2cc7..36bd700576 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -88,11 +88,14 @@ my %pgdump_runs = (
'postgres',
],
# Give coverage for manually compressed blob.toc files during restore.
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-f',
- "$tempdir/compression_gzip_directory_format/blobs.toc",
- ],
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ },
restore_cmd => [
'pg_restore',
"--file=$tempdir/compression_gzip_directory_format.sql",
@@ -108,12 +111,15 @@ my %pgdump_runs = (
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
- # Give coverage for manually compressed blob.toc files during restore.
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-f',
- "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
- ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/toc.dat",
+ ],
+ },
restore_cmd => [
'pg_restore',
'--jobs=3',
@@ -131,12 +137,96 @@ my %pgdump_runs = (
'postgres',
],
# Decompress the generated file to run through the tests
- compress_cmd => [
- $ENV{'GZIP_PROGRAM'},
- '-d',
- "$tempdir/compression_gzip_plain_format.sql.gz",
+ compression => {
+ method => 'gzip',
+ cmd => [
+ $ENV{'GZIP_PROGRAM'},
+ '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
+ },
+ compression_lz4_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=custom', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom_format.sql",
+ "$tempdir/compression_lz4_custom_format.dump",
],
},
+ compression_lz4_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=directory', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{'LZ4'},
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format/toc.dat",
+ "$tempdir/compression_lz4_directory_format/toc.dat.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_directory_format.sql",
+ "$tempdir/compression_lz4_directory_format",
+ ],
+ },
+ compression_lz4_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync', '--jobs=2',
+ '--format=directory', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{'LZ4'},
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc",
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_directory_format_parallel.sql",
+ "$tempdir/compression_lz4_directory_format_parallel",
+ ],
+ },
+ # Check that the output is valid lz4
+ compression_lz4_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--no-sync',
+ '--format=plain', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_plain_format.sql.lz4",
+ 'postgres',
+ ],
+ compression => {
+ method => 'lz4',
+ cmd => [
+ $ENV{LZ4}, '-d', '-f',
+ "$tempdir/compression_lz4_plain_format.sql.lz4",
+ "$tempdir/compression_lz4_plain_format.sql",
+ ],
+ }
+ },
compression_none_dir_format => {
test_key => 'compression',
dump_cmd => [
@@ -145,10 +235,13 @@ my %pgdump_runs = (
"--file=$tempdir/compression_none_dir_format",
'postgres',
],
- glob_match => {
- no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
- match => "$tempdir/compression_none_dir_format/*.dat",
- match_count => 2, # toc.dat and more
+ compression => {
+ method => 'gzip',
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
},
restore_cmd => [
'pg_restore', '-Fd',
@@ -156,6 +249,26 @@ my %pgdump_runs = (
"$tempdir/compression_none_dir_format",
],
},
+ compression_default_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ "--file=$tempdir/compression_default_dir_format",
+ 'postgres',
+ ],
+ compression => {
+ method => 'gzip',
+ glob_match => {
+ match => "$tempdir/compression_default_dir_format/*.dat.gz",
+ match_count => 1, # data
+ },
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_default_dir_format.sql",
+ "$tempdir/compression_default_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -4046,9 +4159,11 @@ command_fails_like(
#########################################
# Run all runs
-my $supports_compression = check_pg_config("#define HAVE_LIBZ 1");
-my $compress_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
- '>', '/dev/null') == 0);
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+my $gzip_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
+ '>', '/dev/null') == 0);
+my $lz4_program_exists = (system_log("$ENV{LZ4}", '-h',
+ '>', '/dev/null') == 0);
foreach my $run (sort keys %pgdump_runs)
{
@@ -4058,21 +4173,27 @@ foreach my $run (sort keys %pgdump_runs)
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
- if (defined($pgdump_runs{$run}->{compress_cmd}))
+ if (defined($pgdump_runs{$run}->{'compression'}))
{
# Skip compression_cmd tests when compression is not supported,
# as the result is uncompressed or the utility program does not
# exist
- next if !$supports_compression || !$compress_program_exists;
+ next if !$supports_gzip && !$supports_lz4;
+
+ my ($compression) = $pgdump_runs{$run}->{'compression'};
- command_ok( \@{ $pgdump_runs{$run}->{compress_cmd} },
- "$run: compression commands");
+ next if ${compression}->{method} eq 'gzip' && (!$gzip_program_exists || !$supports_gzip);
+ next if ${compression}->{method} eq 'lz4' && (!$lz4_program_exists || !$supports_lz4);
+
+ if (defined($compression->{cmd}))
+ {
+ command_ok( \@{ $compression->{cmd} }, "$run: compression commands");
+ }
- if (defined($pgdump_runs{$run}->{glob_match}))
+ if (defined($compression->{glob_match}))
{
- my $match = $pgdump_runs{$run}->{glob_match}->{match};
- my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
- $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
+ my $match = $compression->{glob_match}->{match};
+ my $match_count = $compression->{glob_match}->{match_count};
my @glob_matched = glob $match;
cmp_ok(scalar(@glob_matched), '>=', $match_count,
@@ -4081,9 +4202,9 @@ foreach my $run (sort keys %pgdump_runs)
if (defined($pgdump_runs{$run}->{glob_match}->{no_match}))
{
my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
- my @glob_matched = glob $no_match;
+ my @glob_not_matched = glob $no_match;
- cmp_ok(scalar(@glob_matched), '==', 0,
+ cmp_ok(scalar(@glob_not_matched), '==', 0,
"Expected no file(s) matching $no_match");
}
}
--
2.32.0
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-03-31 02:34 ` Michael Paquier <[email protected]>
2022-04-01 15:06 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-03-31 02:34 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Wed, Mar 30, 2022 at 03:32:55PM +0000, [email protected] wrote:
> On Wednesday, March 30th, 2022 at 7:54 AM, Michael Paquier <[email protected]> wrote:
>> While moving on with 0002, I have noticed the following in
>>
>> _StartBlob():
>> if (AH->compression != 0)
>> sfx = ".gz";
>> else
>> sfx = "";
>>
>> Shouldn't this bit also be simplified, adding a fatal() like the other
>> code paths, for safety?
>
> Agreed. Fixed.
Okay. 0002 looks fine as-is, and I don't mind the extra fatal()
calls. These could be asserts but that's not a big deal one way or
the other. And the cleanup is now applied.
>> + my $compress_program = $ENV{GZIP_PROGRAM};
>>
>> It seems to me that it is enough to rely on {compress_cmd}, hence
>> there should be no need for $compress_program, no?
>
> Maybe not. We don't want to the tests to fail if the utility is not
> installed. That becomes even more evident as more methods are added.
> However I realized that the presence of the environmental variable does
> not guarrantee that the utility is actually installed. In the attached,
> the existance of the utility is based on the return value of system_log().
Hmm. [.. thinks ..] The thing that's itching me here is that you
align the concept of compression with gzip, but that's not going to be
true once more compression options are added to pg_dump, and that
would make $supports_compression and $compress_program_exists
incorrect. Perhaps the right answer would be to rename all that with
a suffix like "_gzip" to make a difference? Or would there be enough
control with a value of "compression_gzip" instead of "compression" in
test_key?
+my $compress_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
+ '>', '/dev/null') == 0);
Do we need this command execution at all? In all the other tests, we
rely on a simple "if (!defined $gzip || $gzip eq '');", so we could do
the same here.
A last thing is that we should perhaps make a clear difference between
the check that looks at if the code has been built with zlib and the
check for the presence of GZIP_PROGRAM, as it can be useful in some
environments to be able to run pg_dump built with zlib, even if the
GZIP_PROGRAM command does not exist (I don't think this can be the
case, but other tests are flexible). As of now, the patch relies on
pg_dump enforcing uncompression if building under --without-zlib even
if --compress/-Z is used, but that also means that those compression
tests overlap with the existing tests in this case. Wouldn't it be
more consistent to check after $supports_compression when executing
the dump command for test_key = "compression[_gzip]"? This would mean
keeping GZIP_PROGRAM as sole check when executing the compression
command.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-04-01 15:06 ` [email protected]
2022-04-05 01:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-04-01 15:06 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Thursday, March 31st, 2022 at 4:34 AM, Michael Paquier <[email protected]> wrote:
> On Wed, Mar 30, 2022 at 03:32:55PM +0000, [email protected] wrote:
> > On Wednesday, March 30th, 2022 at 7:54 AM, Michael Paquier [email protected] wrote:
>
> Okay. 0002 looks fine as-is, and I don't mind the extra fatal()
> calls. These could be asserts but that's not a big deal one way or
> the other. And the cleanup is now applied.
Thank you very much.
> > > + my $compress_program = $ENV{GZIP_PROGRAM};
> > > It seems to me that it is enough to rely on {compress_cmd}, hence
> > > there should be no need for $compress_program, no?
> >
> > Maybe not. We don't want to the tests to fail if the utility is not
> > installed. That becomes even more evident as more methods are added.
> > However I realized that the presence of the environmental variable does
> > not guarrantee that the utility is actually installed. In the attached,
> > the existance of the utility is based on the return value of system_log().
>
> Hmm. [.. thinks ..] The thing that's itching me here is that you
> align the concept of compression with gzip, but that's not going to be
> true once more compression options are added to pg_dump, and that
> would make $supports_compression and $compress_program_exists
> incorrect. Perhaps the right answer would be to rename all that with
> a suffix like "_gzip" to make a difference? Or would there be enough
> control with a value of "compression_gzip" instead of "compression" in
> test_key?
I understand the itch. Indeed when LZ4 is added as compression method, this
block changes slightly. I went with the minimum amount changed. Please find
in 0001 of the attached this variable renamed as $gzip_program_exist. I thought
that as prefix it will match better the already used $ENV{GZIP_PROGRAM}.
> +my $compress_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
> + '>', '/dev/null') == 0);
>
> Do we need this command execution at all? In all the other tests, we
> rely on a simple "if (!defined $gzip || $gzip eq '');", so we could do
> the same here.
You are very correct that we are using the simple version, and that is what
it was included in the previous versions of the current patch. However, I
did notice that the variable is hard-coded in Makefile.global.in and it does
not go through configure. By now, gzip is considered an essential package
in most installations, and this hard-code makes sense. Though I did remove
the utility from my system, (apt remove gzip) and tried the test with the
simple "if (!defined $gzip || $gzip eq '');", which predictably failed. For
this, I went with the system call, it is not too expensive and is rather
reliable.
It is true that the rest of the TAP tests that use this, e.g. in pg_basebackup,
also failed. There is an argument to go simple and I will be happy to revert
to the previous version.
> A last thing is that we should perhaps make a clear difference between
> the check that looks at if the code has been built with zlib and the
> check for the presence of GZIP_PROGRAM, as it can be useful in some
> environments to be able to run pg_dump built with zlib, even if the
> GZIP_PROGRAM command does not exist (I don't think this can be the
> case, but other tests are flexible).
You are very correct. We do that already in the current patch. Note that we skip
the test only when we specifically have to execute a compression command. Not
all compression tests define such command, exactly so that we can test those
cases as well. The point of using an external utility program is in order to
extend the coverage in previously untested yet supported scenarios, e.g. manual
compression of the *.toc files.
Also in the case where it will actually skip the compression command because the
gzip program is not present, it will execute the pg_dump command first.
> As of now, the patch relies on
> pg_dump enforcing uncompression if building under --without-zlib even
> if --compress/-Z is used, but that also means that those compression
> tests overlap with the existing tests in this case. Wouldn't it be
> more consistent to check after $supports_compression when executing
> the dump command for test_key = "compression[_gzip]"? This would mean
> keeping GZIP_PROGRAM as sole check when executing the compression
> command.
I can see the overlap case. Yet, I understand the test_key as serving different
purpose, as it is a key of %tests and %full_runs. I do not expect the database
content of the generated dump to change based on which compression method is used.
In the next round, I can see one explitcly requesting --compress=none to override
defaults. There is a benefit to group the tests for this scenario under the same
test_key, i.e. compression.
Also there will be cases where if the program exists, yet the codebase is compiled
without support for the method. Then compress_cmd or the restore_cmd that follows
will fail. For example, in the plain output, if we try to uncompress the generated
the test will fail with 'gzip: <filename> not in gzip format'. In the directory
format the compress_cmd will compress the *.toc files, but the restore_cmd will
fail because it does not build with support for them.
In the attached version, I propose that the compression_cmd is converted into
a hash. It contains two keys, the program and the arguments. Maybe it is easier
to read than before or than simply grabbing the first element of the array.
Cheers,
//Georgios
Attachments:
[text/x-patch] v5-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch (5.9K, ../../syL-7HopUnzsy_Trvi6DON6tGwrcpujr-LhXo26AJ8AoP1fiGW66H33bMkPYU7tlu5HoqaSTt5FBeI6Ew1RLxnvTKy4-YZtBUJPgFMw3bQM=@pm.me/2-v5-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch)
download | inline diff:
From 093f9104e1e50b777e05f24665c7c1125d97ace3 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 1 Apr 2022 13:15:14 +0000
Subject: [PATCH v5 1/3] Extend compression coverage for pg_dump, pg_restore
---
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/t/002_pg_dump.pl | 110 +++++++++++++++++++++++++++++++
2 files changed, 112 insertions(+)
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 302f7e02d6..2f524b09bf 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
+export GZIP_PROGRAM=$(GZIP)
+
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index af5d6fa5a3..134cc0618b 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -26,6 +26,13 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# specified and is pulled from $PGPORT, which is set by the
# PostgreSQL::Test::Cluster system.
#
+# compress_cmd is the utility command for (de)compression, if any.
+# Note that this should generally be used on pg_dump's output
+# either to generate a text file to run the through the tests, or
+# to test pg_restore's ability to parse manually compressed files
+# that otherwise pg_dump does not compress on it's own
+# (e.g. *.toc).
+#
# restore_cmd is the pg_restore command to run, if any. Note
# that this should generally be used when the pg_dump goes to
# a non-text file and that the restore can then be used to
@@ -54,6 +61,88 @@ my %pgdump_runs = (
"$tempdir/binary_upgrade.dump",
],
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=custom', '--compress=1',
+ "--file=$tempdir/compression_gzip_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_custom_format.sql",
+ "$tempdir/compression_gzip_custom_format.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--format=directory', '--compress=1',
+ "--file=$tempdir/compression_gzip_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => {
+ program => $ENV{'GZIP_PROGRAM'},
+ args => [
+ '-f',
+ "$tempdir/compression_gzip_directory_format/blobs.toc",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_directory_format.sql",
+ "$tempdir/compression_gzip_directory_format",
+ ],
+ },
+
+ compression_gzip_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=6',
+ "--file=$tempdir/compression_gzip_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => {
+ program => $ENV{'GZIP_PROGRAM'},
+ args => [
+ '-f',
+ "$tempdir/compression_gzip_directory_format_parallel/blobs.toc",
+ ]
+ },
+ restore_cmd => [
+ 'pg_restore',
+ '--jobs=3',
+ "--file=$tempdir/compression_gzip_directory_format_parallel.sql",
+ "$tempdir/compression_gzip_directory_format_parallel",
+ ],
+ },
+
+ # Check that the output is valid gzip
+ compression_gzip_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '-Z1',
+ "--file=$tempdir/compression_gzip_plain_format.sql.gz",
+ 'postgres',
+ ],
+ # Decompress the generated file to run through the tests
+ compress_cmd => {
+ program => $ENV{'GZIP_PROGRAM'},
+ args => [
+ '-d',
+ "$tempdir/compression_gzip_plain_format.sql.gz",
+ ],
+ },
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -424,6 +513,7 @@ my %full_runs = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3098,6 +3188,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_test_table => 1,
@@ -3171,6 +3262,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3941,6 +4033,9 @@ command_fails_like(
#########################################
# Run all runs
+my $supports_gzip_compression = check_pg_config("#define HAVE_LIBZ 1");
+my $gzip_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
+ '>', '/dev/null') == 0);
foreach my $run (sort keys %pgdump_runs)
{
@@ -3950,6 +4045,21 @@ foreach my $run (sort keys %pgdump_runs)
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
+ if ($pgdump_runs{$run}->{compress_cmd})
+ {
+ my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd};
+
+ # Skip compression_cmd tests when compression is not supported,
+ # as the result is uncompressed or the utility program does not
+ # exist
+ next if !$supports_gzip_compression;
+ next if $compress_cmd->{program} eq "$ENV{GZIP_PROGRAM}" &&
+ !$gzip_program_exists;
+
+ my @full_cmd = ($compress_cmd->{program}, @{ $compress_cmd->{args} });
+ command_ok(\@full_cmd, "$run: compression commands");
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.32.0
[text/x-patch] v5-0003-Add-LZ4-compression-in-pg_-dump-restore.patch (43.1K, ../../syL-7HopUnzsy_Trvi6DON6tGwrcpujr-LhXo26AJ8AoP1fiGW66H33bMkPYU7tlu5HoqaSTt5FBeI6Ew1RLxnvTKy4-YZtBUJPgFMw3bQM=@pm.me/3-v5-0003-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 39dfbf28e781a4870a20110942bf6a1586b837b6 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 1 Apr 2022 13:15:27 +0000
Subject: [PATCH v5 3/3] Add LZ4 compression in pg_{dump|restore}
Within compress_io.{c,h} there are two distinct APIs exposed, the streaming API
and a file API. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
In the later case, the API is using an opaque wrapper around a file stream,
which aquired via fopen() or gzopen() respectively. It would then provide
wrappers around fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(); or
their gz equivallents. However the LZ4F api does not provide this functionality.
So this has been implemented localy.
In order to maintain the API compatibility a new structure LZ4File is
introduced. It is responsible for keeping state and any yet unused generated
content. The later is required when the generated decompressed output, exceeds
the caller's buffer capacity.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression is
still stored in the dump, though admittedly is of no apparent use.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_io.c | 738 ++++++++++++++++++++++++---
src/bin/pg_dump/pg_backup.h | 3 +-
src/bin/pg_dump/pg_backup_archiver.c | 71 ++-
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_dump.c | 12 +-
src/bin/pg_dump/t/001_basic.pl | 4 +-
src/bin/pg_dump/t/002_pg_dump.pl | 104 +++-
9 files changed, 852 insertions(+), 107 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 992b7312df..68a7c6a3bf 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 2f524b09bf..2864ccabb9 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 630f9e4b18..790f225ec4 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -56,6 +58,14 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBLZ4
+#include "lz4.h"
+#include "lz4frame.h"
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -69,9 +79,9 @@ struct CompressorState
#ifdef HAVE_LIBZ
z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
#endif
+ void *outbuf;
+ size_t outsize;
};
/* Routines that support zlib compressed data I/O */
@@ -85,6 +95,15 @@ static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
#endif
+/* Routines that support LZ4 compressed data I/O */
+#ifdef HAVE_LIBLZ4
+static void InitCompressorLZ4(CompressorState *cs);
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+#endif
+
/* Routines that support uncompressed data I/O */
static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
@@ -103,6 +122,11 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
+#ifndef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ fatal("not built with LZ4 support");
+#endif
+
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
@@ -115,6 +139,10 @@ AllocateCompressor(CompressionMethod compressionMethod,
if (compressionMethod == COMPRESSION_GZIP)
InitCompressorZlib(cs, compressionLevel);
#endif
+#ifdef HAVE_LIBLZ4
+ if (compressionMethod == COMPRESSION_LZ4)
+ InitCompressorLZ4(cs);
+#endif
return cs;
}
@@ -137,6 +165,13 @@ ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
ReadDataFromArchiveZlib(AH, readF);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ReadDataFromArchiveLZ4(AH, readF);
+#else
+ fatal("not built with lz4 support");
#endif
break;
default:
@@ -159,6 +194,13 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ WriteDataToArchiveLZ4(AH, cs, data, dLen);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -183,6 +225,13 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
EndCompressorZlib(AH, cs);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ EndCompressorLZ4(AH, cs);
+#else
+ fatal("not built with lz4 support");
#endif
break;
case COMPRESSION_NONE:
@@ -213,20 +262,20 @@ InitCompressorZlib(CompressorState *cs, int level)
zp->opaque = Z_NULL;
/*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
+ * outsize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to append a
* trailing zero byte to the zlib output.
*/
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
+ cs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ cs->outsize = ZLIB_OUT_SIZE;
if (deflateInit(zp, level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
/* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = cs->outbuf;
+ zp->avail_out = cs->outsize;
}
static void
@@ -243,7 +292,7 @@ EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
if (deflateEnd(zp) != Z_OK)
fatal("could not close compression stream: %s", zp->msg);
- free(cs->zlibOut);
+ free(cs->outbuf);
free(cs->zp);
}
@@ -251,7 +300,7 @@ static void
DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
{
z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
+ void *out = cs->outbuf;
int res = Z_OK;
while (cs->zp->avail_in != 0 || flush)
@@ -259,7 +308,7 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
if (res == Z_STREAM_ERROR)
fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
+ if ((flush && (zp->avail_out < cs->outsize))
|| (zp->avail_out == 0)
|| (zp->avail_in != 0)
)
@@ -269,18 +318,18 @@ DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
* chunk is the EOF marker in the custom format. This should never
* happen but...
*/
- if (zp->avail_out < cs->zlibOutSize)
+ if (zp->avail_out < cs->outsize)
{
/*
* Any write function should do its own error checking but to
* make sure we do a check here as well...
*/
- size_t len = cs->zlibOutSize - zp->avail_out;
+ size_t len = cs->outsize - zp->avail_out;
- cs->writeF(AH, out, len);
+ cs->writeF(AH, (char *)out, len);
}
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
+ zp->next_out = out;
+ zp->avail_out = cs->outsize;
}
if (res == Z_STREAM_END)
@@ -364,6 +413,71 @@ ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
}
#endif /* HAVE_LIBZ */
+#ifdef HAVE_LIBLZ4
+static void
+InitCompressorLZ4(CompressorState *cs)
+{
+ /* Will be lazy init'd */
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ pg_free(cs->outbuf);
+
+ cs->outbuf = NULL;
+ cs->outsize = 0;
+}
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, ReadFunc readF)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = (4 * 1024) + 1;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const char *data, size_t dLen)
+{
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > cs->outsize)
+ {
+ cs->outbuf = pg_realloc(cs->outbuf, requiredsize);
+ cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, cs->outbuf,
+ dLen, cs->outsize);
+
+ if (compressed <= 0)
+ fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, cs->outbuf, compressed);
+}
+#endif /* HAVE_LIBLZ4 */
/*
* Functions for uncompressed output.
@@ -400,9 +514,36 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*----------------------
*/
+#ifdef HAVE_LIBLZ4
/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
+ * State needed for LZ4 (de)compression using the cfp API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+#endif
+
+/*
+ * cfp represents an open stream, wrapping the underlying FILE, gzFile
+ * pointer, or LZ4File pointer. This is opaque to the callers.
*/
struct cfp
{
@@ -410,9 +551,7 @@ struct cfp
void *fp;
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -424,26 +563,380 @@ free_keep_errno(void *p)
errno = save_errno;
}
+#ifdef HAVE_LIBLZ4
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(LZ4File *fs)
+{
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_error(LZ4File *fs)
+{
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File *fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen, &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File *fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ readlen = p - fs->overflowbuf + 1; /* Include the line terminating char */
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read(LZ4File *fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *)readbuf;
+ rend = (char *)readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr
+ * if the eol flag is set, either skip if one already found or fill up to EOL
+ * if present in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size -1 -dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true && (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t)(p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *)ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf, fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int)dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static int
+LZ4File_write(LZ4File *fs, const void *ptr, int size)
+{
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fgetc() and gzgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(LZ4File *fs)
+{
+ unsigned char c;
+
+ if (LZ4File_read(fs, &c, 1, false) != 1)
+ return EOF;
+
+ return c;
+}
+
+/*
+ * fgets() and gzgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(LZ4File *fs, char *buf, int len)
+{
+ size_t dsize;
+
+ dsize = LZ4File_read(fs, buf, len, true);
+ if (dsize < 0)
+ fatal("failed to read from archive %s", LZ4File_error(fs));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return buf;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(LZ4File *fs)
+{
+ FILE *fp;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ fatal("failed to end compression: %s", LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ fatal("failed to end decompression: %s", LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+#endif
+
/*
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * this will open either "foo" or "foo.gz" or "foo.lz4", trying in that order.
*
* On failure, return NULL with an error code in errno.
+ *
*/
cfp *
cfopen_read(const char *path, const char *mode)
{
- cfp *fp;
+ cfp *fp = NULL;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
+ else if (hasSuffix(path, ".lz4"))
+ fp = cfopen(path, mode, COMPRESSION_LZ4, 0);
else
-#endif
{
fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
@@ -455,8 +948,19 @@ cfopen_read(const char *path, const char *mode)
fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
+#endif
+#ifdef HAVE_LIBLZ4
+ if (fp == NULL)
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, COMPRESSION_LZ4, 0);
+ free_keep_errno(fname);
+ }
#endif
}
+
return fp;
}
@@ -465,9 +969,13 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * When 'compressionMethod' indicates gzip, a gzip compressed stream is opened,
+ * and 'compressionLevel' is used. The ".gz" suffix is automatically added to
+ * 'path' in that case. The same applies when 'compressionMethod' indicates lz4,
+ * but then the ".lz4" suffix is added instead.
+ *
+ * It is the caller's responsibility to verify that the requested
+ * 'compressionMethod' is supported by the build.
*
* On failure, return NULL with an error code in errno.
*/
@@ -476,23 +984,44 @@ cfopen_write(const char *path, const char *mode,
CompressionMethod compressionMethod,
int compressionLevel)
{
- cfp *fp;
+ cfp *fp = NULL;
- if (compressionMethod == COMPRESSION_NONE)
- fp = cfopen(path, mode, compressionMethod, 0);
- else
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ fp = cfopen(path, mode, compressionMethod, 0);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- char *fname;
+ {
+ char *fname;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compressionMethod, compressionLevel);
- free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
+ fatal("not built with zlib support");
#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ char *fname;
+
+ fname = psprintf("%s.lz4", path);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
+ free_keep_errno(fname);
+ }
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return fp;
}
@@ -509,7 +1038,7 @@ static cfp *
cfopen_internal(const char *path, int fd, const char *mode,
CompressionMethod compressionMethod, int compressionLevel)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
fp->compressionMethod = compressionMethod;
@@ -560,6 +1089,27 @@ cfopen_internal(const char *path, int fd, const char *mode,
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ {
+ LZ4File *lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (fd >= 0)
+ lz4fp->fp = fdopen(fd, mode);
+ else
+ lz4fp->fp = fopen(path, mode);
+ if (lz4fp->fp == NULL)
+ {
+ free_keep_errno(lz4fp);
+ fp = NULL;
+ }
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+ fp->fp = lz4fp;
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -580,8 +1130,8 @@ cfopen(const char *path, const char *mode,
cfp *
cfdopen(int fd, const char *mode,
- CompressionMethod compressionMethod,
- int compressionLevel)
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
}
@@ -617,7 +1167,16 @@ cfread(void *ptr, int size, cfp *fp)
fatal("not built with zlib support");
#endif
break;
-
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_read(fp->fp, ptr, size, false);
+ if (ret != size && !LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s",
+ LZ4File_error(fp->fp));
+#else
+ fatal("not built with LZ4 support");
+#endif
+ break;
default:
fatal("invalid compression method");
break;
@@ -641,6 +1200,13 @@ cfwrite(const void *ptr, int size, cfp *fp)
ret = gzwrite(fp->fp, ptr, size);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_write(fp->fp, ptr, size);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -676,6 +1242,20 @@ cfgetc(cfp *fp)
}
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_getc(fp->fp);
+ if (ret == EOF)
+ {
+ if (!LZ4File_eof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -695,13 +1275,19 @@ cfgets(cfp *fp, char *buf, int len)
{
case COMPRESSION_NONE:
ret = fgets(buf, len, fp->fp);
-
break;
case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
ret = gzgets(fp->fp, buf, len);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_gets(fp->fp, buf, len);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -736,6 +1322,14 @@ cfclose(cfp *fp)
fp->fp = NULL;
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_close(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -764,6 +1358,13 @@ cfeof(cfp *fp)
ret = gzeof(fp->fp);
#else
fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ ret = LZ4File_eof(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
break;
default:
@@ -777,23 +1378,42 @@ cfeof(cfp *fp)
const char *
get_cfp_error(cfp *fp)
{
- if (fp->compressionMethod == COMPRESSION_GZIP)
+ const char *errmsg = NULL;
+
+ switch(fp->compressionMethod)
{
+ case COMPRESSION_NONE:
+ errmsg = strerror(errno);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ {
+ int errnum;
+ errmsg = gzerror(fp->fp, &errnum);
- if (errnum != Z_ERRNO)
- return errmsg;
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+ }
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
+#endif
+ break;
+ case COMPRESSION_LZ4:
+#ifdef HAVE_LIBLZ4
+ errmsg = LZ4File_error(fp->fp);
+#else
+ fatal("not built with LZ4 support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
- return strerror(errno);
+ return errmsg;
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -807,5 +1427,3 @@ hasSuffix(const char *filename, const char *suffix)
suffix,
suffixlen) == 0;
}
-
-#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 7645d3285a..10393f3fc4 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -78,8 +78,9 @@ enum _dumpPreparedQueries
typedef enum _compressionMethod
{
COMPRESSION_INVALID,
+ COMPRESSION_GZIP,
COMPRESSION_NONE,
- COMPRESSION_GZIP
+ COMPRESSION_LZ4
} CompressionMethod;
/* Parameters needed by ConnectDatabase; same for dump and restore */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7d96446f1a..e6f727534b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -353,6 +353,7 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
cfp *sav;
@@ -382,17 +383,27 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compressionMethod == COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compressionMethod != COMPRESSION_NONE &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (AH->compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -2019,6 +2030,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2046,30 +2069,21 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
+ return AH->format;
+#endif
+#ifdef HAVE_LIBLZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
return AH->format;
- }
#endif
fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3681,6 +3695,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
WriteInt(AH, AH->compressionLevel);
+ AH->WriteBytePtr(AH, AH->compressionMethod);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3761,14 +3776,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compressionLevel = Z_DEFAULT_COMPRESSION;
- if (AH->compressionLevel != INT_MIN)
+ if (AH->version >= K_VERS_1_15)
+ AH->compressionMethod = AH->ReadBytePtr(AH);
+ else if (AH->compressionLevel != 0)
+ AH->compressionMethod = COMPRESSION_GZIP;
+
#ifndef HAVE_LIBZ
+ if (AH->compressionMethod == COMPRESSION_GZIP)
+ {
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
-#else
- AH->compressionMethod = COMPRESSION_GZIP;
+ AH->compressionMethod = COMPRESSION_NONE;
+ AH->compressionLevel = 0;
+ }
#endif
-
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 837e9d73f5..037bfcf913 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,11 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compressionMethod in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 97ac17ebff..5351c71d2b 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1002,7 +1002,7 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ printf(_(" -Z, --compress=[gzip,lz4,none][:LEVEL] or [LEVEL]\n"
" compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
@@ -1271,11 +1271,13 @@ parse_compression_method(const char *method,
if (pg_strcasecmp(method, "gzip") == 0)
*compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "lz4") == 0)
+ *compressionMethod = COMPRESSION_LZ4;
else if (pg_strcasecmp(method, "none") == 0)
*compressionMethod = COMPRESSION_NONE;
else
{
- pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ pg_log_error("invalid compression method \"%s\" (gzip, lz4, none)", method);
res = false;
}
@@ -1346,10 +1348,10 @@ parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
if (!res)
return res;
- /* one can set level when method is gzip */
- if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ /* one can set level when a compression method is set */
+ if (*compressionMethod == COMPRESSION_NONE && *compressLevel != INT_MIN)
{
- pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is set");
return false;
}
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index d7a52ac1a8..0077288388 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,12 +122,12 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
[ 'pg_dump', '--compress', 'none:1' ],
- qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is set\E/,
'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 05a4b0756b..acf4ffa2a9 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -143,6 +143,85 @@ my %pgdump_runs = (
],
},
},
+ compression_lz4_custom_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=custom', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_custom_format.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom_format.sql",
+ "$tempdir/compression_lz4_custom_format.dump",
+ ],
+ },
+ compression_lz4_directory_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ '--format=directory', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_directory_format",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed toc.dat files during restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format/toc.dat",
+ "$tempdir/compression_lz4_directory_format/toc.dat.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_directory_format.sql",
+ "$tempdir/compression_lz4_directory_format",
+ ],
+ },
+ compression_lz4_directory_format_parallel => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '--no-sync', '--jobs=2',
+ '--format=directory', '--compress=lz4:9',
+ "--file=$tempdir/compression_lz4_directory_format_parallel",
+ 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc",
+ "$tempdir/compression_lz4_directory_format_parallel/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_directory_format_parallel.sql",
+ "$tempdir/compression_lz4_directory_format_parallel",
+ ],
+ },
+ # Check that the output is valid lz4
+ compression_lz4_plain_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump',
+ '--no-sync',
+ '--format=plain', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_plain_format.sql.lz4",
+ 'postgres',
+ ],
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain_format.sql.lz4",
+ "$tempdir/compression_lz4_plain_format.sql",
+ ],
+ }
+ },
compression_none_dir_format => {
test_key => 'compression',
dump_cmd => [
@@ -162,6 +241,23 @@ my %pgdump_runs = (
"$tempdir/compression_none_dir_format",
],
},
+ compression_default_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ "--file=$tempdir/compression_default_dir_format",
+ 'postgres',
+ ],
+ glob_match => {
+ match => "$tempdir/compression_default_dir_format/*.dat.gz",
+ match_count => 1, # data
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_default_dir_format.sql",
+ "$tempdir/compression_default_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -4055,6 +4151,8 @@ command_fails_like(
my $supports_gzip_compression = check_pg_config("#define HAVE_LIBZ 1");
my $gzip_program_exists = (system_log("$ENV{GZIP_PROGRAM}", '-h',
'>', '/dev/null') == 0);
+my $lz4_program_exists = (system_log("$ENV{LZ4}", '-h',
+ '>', '/dev/null') == 0);
foreach my $run (sort keys %pgdump_runs)
{
@@ -4071,9 +4169,11 @@ foreach my $run (sort keys %pgdump_runs)
# Skip compression_cmd tests when compression is not supported,
# as the result is uncompressed or the utility program does not
# exist
- next if !$supports_gzip_compression;
+ next if !$supports_gzip_compression && !$supports_lz4;
next if $compress_cmd->{program} eq "$ENV{GZIP_PROGRAM}" &&
!$gzip_program_exists;
+ next if $compress_cmd->{program} eq "$ENV{LZ4}" &&
+ !$lz4_program_exists;
my @full_cmd = ($compress_cmd->{program}, @{ $compress_cmd->{args} });
command_ok(\@full_cmd, "$run: compression commands");
@@ -4083,7 +4183,7 @@ foreach my $run (sort keys %pgdump_runs)
if ($pgdump_runs{$run}->{glob_match})
{
# Skip compression_cmd tests when compression is not supported
- next if !$supports_gzip_compression;
+ next if !$supports_gzip_compression && !$supports_lz4;
my $match = $pgdump_runs{$run}->{glob_match}->{match};
my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
--
2.32.0
[text/x-patch] v5-0002-Prepare-pg_dump-for-additional-compression-method.patch (52.8K, ../../syL-7HopUnzsy_Trvi6DON6tGwrcpujr-LhXo26AJ8AoP1fiGW66H33bMkPYU7tlu5HoqaSTt5FBeI6Ew1RLxnvTKy4-YZtBUJPgFMw3bQM=@pm.me/4-v5-0002-Prepare-pg_dump-for-additional-compression-method.patch)
download | inline diff:
From b6c231f35b5163b755d2a79cc6d9b12920406731 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 1 Apr 2022 13:15:22 +0000
Subject: [PATCH v5 2/3] Prepare pg_dump for additional compression methods
This commmit does the heavy lifting required for additional compression methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
Furthermore, compression was chosen based on the value of the level passed
as an argument during the invocation of pg_dump or some hardcoded defaults. This
does not scale for more than one compression methods. Now the method used for
compression can be explicitly requested during command invocation, or set during
hardcoded defaults. Then it is stored in the relevant structs and passed in the
relevant functions, along side compression level which has lost it's special
meaning. The method for compression is not yet stored in the actual archive.
This is done in the next commit which does introduce a new method.
The previously named CompressionAlgorithm enum is changed for
CompressionMethod so that it matches better similar variables found through out
the code base.
In a fashion similar to the binary for pg_basebackup, the method for compression
is passed using the already existing -Z/--compress parameter of pg_dump. The
legacy format and behaviour is maintained. Additionally, the user can explicitly
pass a requested method and optionaly the level to be used after a semicolon,
e.g. --compress=gzip:6
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 416 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 32 +-
src/bin/pg_dump/pg_backup.h | 14 +-
src/bin/pg_dump/pg_backup_archiver.c | 171 +++++------
src/bin/pg_dump/pg_backup_archiver.h | 46 +--
src/bin/pg_dump/pg_backup_custom.c | 11 +-
src/bin/pg_dump/pg_backup_directory.c | 12 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 155 ++++++++--
src/bin/pg_dump/t/001_basic.pl | 14 +-
src/bin/pg_dump/t/002_pg_dump.pl | 49 ++-
src/tools/pgindent/typedefs.list | 2 +-
13 files changed, 607 insertions(+), 357 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2f0042fd96..992b7312df 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 9077fdb74d..630f9e4b18 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ CompressionMethod compressionMethod;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compressionMethod == COMPRESSION_GZIP)
fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compressionMethod = compressionMethod;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (compressionMethod == COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compressionLevel);
#endif
return cs;
@@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, CompressionMethod compressionMethod,
+ int compressionLevel, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compressionMethod)
{
+ case COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- fatal("not built with zlib support");
+ fatal("not built with zlib support");
#endif
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +152,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compressionMethod)
{
- case COMPR_ALG_LIBZ:
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compressionMethod)
+ {
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ CompressionMethod compressionMethod;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -455,18 +441,18 @@ cfopen_read(const char *path, const char *mode)
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, COMPRESSION_GZIP, 0);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ fp = cfopen(path, mode, COMPRESSION_NONE, 0);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, COMPRESSION_GZIP, 0);
free_keep_errno(fname);
}
#endif
@@ -486,19 +472,21 @@ cfopen_read(const char *path, const char *mode)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compressionMethod == COMPRESSION_NONE)
+ fp = cfopen(path, mode, compressionMethod, 0);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
free_keep_errno(fname);
#else
fatal("not built with zlib support");
@@ -509,60 +497,94 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compressionMethod' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ CompressionMethod compressionMethod, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compressionMethod = compressionMethod;
+
+ switch (compressionMethod)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(path, -1, mode, compressionMethod, compressionLevel);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel)
+{
+ return cfopen_internal(NULL, fd, mode, compressionMethod, compressionLevel);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +594,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +656,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compressionMethod)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- fatal("could not read from input file: %s", strerror(errno));
- else
- fatal("could not read from input file: end of file");
- }
- }
- else
+ case COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile)fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ fatal("could not read from input file: %s", strerror(errno));
+ else
+ fatal("could not read from input file: end of file");
+ }
+#else
+ fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +689,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compressionMethod)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compressionMethod)
+ {
+ case COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compressionMethod == COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..b8b366616c 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -17,16 +17,17 @@
#include "pg_backup_archiver.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#else
+/* this is just the redefinition of a libz constant */
+#define Z_DEFAULT_COMPRESSION (-1)
+#endif
+
/* Initial buffer sizes used in zlib compression. */
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +47,12 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(CompressionMethod compressionMethod,
+ int compressionLevel,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ CompressionMethod compressionMethod,
+ int compressionLevel,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +61,16 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+extern cfp *cfdopen(int fd, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index fcc5f6bd05..7645d3285a 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -75,6 +75,13 @@ enum _dumpPreparedQueries
NUM_PREP_QUERIES /* must be last */
};
+typedef enum _compressionMethod
+{
+ COMPRESSION_INVALID,
+ COMPRESSION_NONE,
+ COMPRESSION_GZIP
+} CompressionMethod;
+
/* Parameters needed by ConnectDatabase; same for dump and restore */
typedef struct _connParams
{
@@ -143,7 +150,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ CompressionMethod compressionMethod;
+ int compressionLevel;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +311,9 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compression,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index d41a99d6ea..7d96446f1a 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,9 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +94,11 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod,
+ int compressionLevel);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +237,15 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt,
+ compressionMethod, compressionLevel,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +255,8 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, COMPRESSION_NONE, 0, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
@@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compressionMethod == COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +459,9 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename,
+ ropt->compressionMethod, ropt->compressionLevel);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -740,7 +741,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compressionMethod != COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -970,6 +971,7 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compressionMethod = COMPRESSION_NONE;
return opts;
}
@@ -1117,13 +1119,13 @@ PrintTOCSummary(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, COMPRESSION_NONE, 0);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1132,7 +1134,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compressionLevel);
switch (AH->format)
{
@@ -1486,60 +1488,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ CompressionMethod compressionMethod, int compressionLevel)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compressionMethod, compressionLevel);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compressionMethod, compressionLevel);
if (!AH->OF)
{
@@ -1550,33 +1527,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *)AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1700,22 +1668,16 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
-
- else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're
+ * connected then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ else
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2200,7 +2162,9 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const CompressionMethod compressionMethod,
+ const int compressionLevel,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2251,14 +2215,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compressionMethod = compressionMethod;
+ AH->compressionLevel = compressionLevel;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, COMPRESSION_NONE, 0);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2266,7 +2230,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compressionMethod != COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3716,7 +3680,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compressionLevel);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3790,18 +3754,21 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compressionLevel = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compressionLevel = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compressionLevel = Z_DEFAULT_COMPRESSION;
+ if (AH->compressionLevel != INT_MIN)
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+#else
+ AH->compressionMethod = COMPRESSION_GZIP;
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 540d4f6a83..837e9d73f5 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,17 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ CompressionMethod compressionMethod; /* Requested method for compression */
+ int compressionLevel; /*---------
+ * Requested level of compression for method.
+ * Possible values for compression:
+ * INT_MIN when no compression method is
+ * requested.
+ * -1 Z_DEFAULT_COMPRESSION for gzip
+ * compression.
+ * 1-9 levels for gzip compression.
+ *---------
+ */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..7f38ea9cd5 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -377,7 +379,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compressionMethod,
+ AH->compressionLevel,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +570,8 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compressionMethod, AH->compressionLevel,
+ _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7f4e340dea..0e60b447de 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod,
+ AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W,
+ COMPRESSION_NONE, 0);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -644,7 +647,7 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", COMPRESSION_NONE, 0);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +665,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compressionMethod, AH->compressionLevel);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 2491a091b9..b25b641caa 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compressionMethod == COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -801,7 +802,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -889,7 +889,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compressionMethod != COMPRESSION_NONE)
fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 535b160165..97ac17ebff 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -55,6 +55,7 @@
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
#include "common/connect.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -163,6 +164,9 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression_option(const char *opt,
+ CompressionMethod *compressionMethod,
+ int *compressLevel);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -336,8 +340,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ int compressLevel = INT_MIN;
+ CompressionMethod compressionMethod = COMPRESSION_INVALID;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -557,9 +562,9 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression_option(optarg, &compressionMethod,
+ &compressLevel))
exit_nicely(1);
break;
@@ -689,23 +694,21 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /* Set default compressionMethod unless one already set by the user */
+ if (compressionMethod == COMPRESSION_INVALID)
{
+ compressionMethod = COMPRESSION_NONE;
+
#ifdef HAVE_LIBZ
+ /* Custom and directory formats are compressed by default (zlib) */
if (archiveFormat == archCustom || archiveFormat == archDirectory)
+ {
+ compressionMethod = COMPRESSION_GZIP;
compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ }
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -718,8 +721,9 @@ main(int argc, char **argv)
fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat,
+ compressionMethod, compressLevel,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -950,10 +954,8 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compressionLevel = compressLevel;
+ ropt->compressionMethod = compressionMethod;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -1000,7 +1002,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=[gzip,none][:LEVEL] or [LEVEL]\n"
+ " compress output with given method or level\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1260,6 +1263,116 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+static bool
+parse_compression_method(const char *method,
+ CompressionMethod *compressionMethod)
+{
+ bool res = true;
+
+ if (pg_strcasecmp(method, "gzip") == 0)
+ *compressionMethod = COMPRESSION_GZIP;
+ else if (pg_strcasecmp(method, "none") == 0)
+ *compressionMethod = COMPRESSION_NONE;
+ else
+ {
+ pg_log_error("invalid compression method \"%s\" (gzip, none)", method);
+ res = false;
+ }
+
+ return res;
+}
+
+/*
+ * Interprets a compression option of the format 'method[:LEVEL]' of legacy just
+ * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level
+ * are returned in *compressionMethod and *compressionLevel. In case of error,
+ * the function returns false and then the values of *compression{Method,Level}
+ * are not to be trusted.
+ */
+static bool
+parse_compression_option(const char *opt, CompressionMethod *compressionMethod,
+ int *compressLevel)
+{
+ char *method;
+ const char *sep;
+ int methodlen;
+ bool supports_compression = true;
+ bool res = true;
+
+ /* find the separator if exists */
+ sep = strchr(opt, ':');
+
+ /*
+ * If there is no separator, then it is either a legacy format, or only the
+ * method has been passed.
+ */
+ if (!sep)
+ {
+ if (strspn(opt, "-0123456789") == strlen(opt))
+ {
+ res = option_parse_int(opt, "-Z/--compress", 0, 9, compressLevel);
+ *compressionMethod = (*compressLevel > 0) ? COMPRESSION_GZIP :
+ COMPRESSION_NONE;
+ }
+ else
+ res = parse_compression_method(opt, compressionMethod);
+ }
+ else
+ {
+ /* otherwise, it should be method:LEVEL */
+ methodlen = sep - opt + 1;
+ method = pg_malloc0(methodlen);
+ snprintf(method, methodlen, "%.*s", methodlen - 1, opt);
+
+ res = parse_compression_method(method, compressionMethod);
+ if (res)
+ {
+ sep++;
+ if (*sep == '\0')
+ {
+ pg_log_error("no level defined for compression \"%s\"", method);
+ pg_free(method);
+ res = false;
+ }
+ else
+ {
+ res = option_parse_int(sep, "-Z/--compress [LEVEL]", 1, 9,
+ compressLevel);
+ }
+ }
+ }
+
+ /* if there is an error, there is no need to check further */
+ if (!res)
+ return res;
+
+ /* one can set level when method is gzip */
+ if (*compressionMethod != COMPRESSION_GZIP && *compressLevel != INT_MIN)
+ {
+ pg_log_error("can only specify -Z/--compress [LEVEL] when method is gzip");
+ return false;
+ }
+
+ /* verify that the requested compression is supported */
+#ifndef HAVE_LIBZ
+ if (*compressionMethod == COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef HAVE_LIBLZ4
+ if (*compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ *compressionMethod = COMPRESSION_NONE;
+ *compressLevel = INT_MIN;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 65e6c01fed..d7a52ac1a8 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -120,6 +120,16 @@ command_fails_like(
qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/,
'pg_restore: cannot specify both --single-transaction and multiple jobs');
+command_fails_like(
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: can only specify -Z\/--compress [LEVEL] when method is gzip\E/,
+ 'pg_dump: can only specify -Z/--compress [LEVEL] when method is gzip');
+
command_fails_like(
[ 'pg_dump', '-Z', '-1' ],
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
@@ -128,7 +138,7 @@ command_fails_like(
if (check_pg_config("#define HAVE_LIBZ 1"))
{
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
}
@@ -136,7 +146,7 @@ else
{
# --jobs > 1 forces an error with tar format.
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
+ [ 'pg_dump', '--compress', 'gzip:1', '--format', 'tar', '-j3' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: compression not available in this installation');
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 134cc0618b..05a4b0756b 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -83,7 +83,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_directory_format",
'postgres',
],
@@ -106,7 +106,7 @@ my %pgdump_runs = (
test_key => 'compression',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=6',
+ '--format=directory', '--compress=gzip:6',
"--file=$tempdir/compression_gzip_directory_format_parallel",
'postgres',
],
@@ -143,6 +143,25 @@ my %pgdump_runs = (
],
},
},
+ compression_none_dir_format => {
+ test_key => 'compression',
+ dump_cmd => [
+ 'pg_dump', '-Fd',
+ '--compress=none',
+ "--file=$tempdir/compression_none_dir_format",
+ 'postgres',
+ ],
+ glob_match => {
+ no_match => "$tempdir/compression_none_dir_format/*.dat.gz",
+ match => "$tempdir/compression_none_dir_format/*.dat",
+ match_count => 2, # toc.dat and more
+ },
+ restore_cmd => [
+ 'pg_restore', '-Fd',
+ "--file=$tempdir/compression_none_dir_format.sql",
+ "$tempdir/compression_none_dir_format",
+ ],
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -242,7 +261,7 @@ my %pgdump_runs = (
defaults_dir_format => {
test_key => 'defaults',
dump_cmd => [
- 'pg_dump', '-Fd',
+ 'pg_dump', '-Fd',
"--file=$tempdir/defaults_dir_format", 'postgres',
],
restore_cmd => [
@@ -4058,6 +4077,30 @@ foreach my $run (sort keys %pgdump_runs)
my @full_cmd = ($compress_cmd->{program}, @{ $compress_cmd->{args} });
command_ok(\@full_cmd, "$run: compression commands");
+
+ }
+
+ if ($pgdump_runs{$run}->{glob_match})
+ {
+ # Skip compression_cmd tests when compression is not supported
+ next if !$supports_gzip_compression;
+
+ my $match = $pgdump_runs{$run}->{glob_match}->{match};
+ my $match_count = defined($pgdump_runs{$run}->{glob_match}->{match_count}) ?
+ $pgdump_runs{$run}->{glob_match}->{match_count} : 1;
+ my @glob_matched = glob $match;
+
+ cmp_ok(scalar(@glob_matched), '>=', $match_count,
+ "Expected at least $match_count file(s) matching $match");
+
+ if ($pgdump_runs{$run}->{glob_match}->{no_match})
+ {
+ my $no_match = $pgdump_runs{$run}->{glob_match}->{no_match};
+ my @glob_matched = glob $no_match;
+
+ cmp_ok(scalar(@glob_matched), '==', 0,
+ "Expected no file(s) matching $no_match");
+ }
}
if ($pgdump_runs{$run}->{restore_cmd})
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 72fafb795b..5ac8c156a9 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -412,7 +412,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
-CompressionAlgorithm
+CompressionMethod
CompressorState
ComputeXidHorizonsResult
ConditionVariable
--
2.32.0
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-04-01 15:06 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-04-05 01:34 ` Michael Paquier <[email protected]>
2022-04-05 10:55 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2025-10-12 17:24 ` Re: Add LZ4 compression in pg_dump Tom Lane <[email protected]>
0 siblings, 2 replies; 67+ messages in thread
From: Michael Paquier @ 2022-04-05 01:34 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Fri, Apr 01, 2022 at 03:06:40PM +0000, [email protected] wrote:
> I understand the itch. Indeed when LZ4 is added as compression method, this
> block changes slightly. I went with the minimum amount changed. Please find
> in 0001 of the attached this variable renamed as $gzip_program_exist. I thought
> that as prefix it will match better the already used $ENV{GZIP_PROGRAM}.
Hmm. I have spent some time on that, and upon review I really think
that we should skip the tests marked as dedicated to the gzip
compression entirely if the build is not compiled with this option,
rather than letting the code run a dump for nothing in some cases,
relying on the default to uncompress the contents in others. In the
latter case, it happens that we have already some checks like
defaults_custom_format, but you already mentioned that.
We should also skip the later parts of the tests if the compression
program does not exist as we rely on it, but only if the command does
not exist. This will count for LZ4.
> I can see the overlap case. Yet, I understand the test_key as serving different
> purpose, as it is a key of %tests and %full_runs. I do not expect the database
> content of the generated dump to change based on which compression method is used.
Contrary to the current LZ4 tests in pg_dump, what we have here is a
check for a command-level run and not a data-level check. So what's
introduced is a new concept, and we need a new way to control if the
tests should be entirely skipped or not, particularly if we finish by
not using test_key to make the difference. Perhaps the best way to
address that is to have a new keyword in the $runs structure. The
attached defines a new compile_option, that can be completed later for
new compression methods introduced in the tests. So the idea is to
mark all the tests related to compression with the same test_key, and
the tests can be skipped depending on what compile_option requires.
> In the attached version, I propose that the compression_cmd is converted into
> a hash. It contains two keys, the program and the arguments. Maybe it is easier
> to read than before or than simply grabbing the first element of the array.
Splitting the program and its arguments makes sense.
At the end I am finishing with the attached. I also saw an overlap
with the addition of --jobs for the directory format vs not using the
option, so I have removed the case where --jobs was not used in the
directory format.
--
Michael
Attachments:
[text/x-diff] v6-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch (6.2K, ../../[email protected]/2-v6-0001-Extend-compression-coverage-for-pg_dump-pg_restor.patch)
download | inline diff:
From b705f77862c9e41a149ce078df638032903bce3d Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Tue, 5 Apr 2022 10:30:06 +0900
Subject: [PATCH v6] Extend compression coverage for pg_dump, pg_restore
---
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/t/002_pg_dump.pl | 93 +++++++++++++++++++++++++++++++-
2 files changed, 93 insertions(+), 2 deletions(-)
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 302f7e02d6..2f524b09bf 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
+export GZIP_PROGRAM=$(GZIP)
+
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index af5d6fa5a3..dda3649373 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -20,12 +20,22 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# test_key indicates that a given run should simply use the same
# set of like/unlike tests as another run, and which run that is.
#
+# compile_option indicates if the commands run depend on a compilation
+# option, if any. This controls if tests are skipped if a dependency
+# is not satisfied.
+#
# dump_cmd is the pg_dump command to run, which is an array of
# the full command and arguments to run. Note that this is run
# using $node->command_ok(), so the port does not need to be
# specified and is pulled from $PGPORT, which is set by the
# PostgreSQL::Test::Cluster system.
#
+# compress_cmd is the utility command for (de)compression, if any.
+# Note that this should generally be used on pg_dump's output
+# either to generate a text file to run the through the tests, or
+# to test pg_restore's ability to parse manually compressed files
+# that otherwise pg_dump does not compress on its own (e.g. *.toc).
+#
# restore_cmd is the pg_restore command to run, if any. Note
# that this should generally be used when the pg_dump goes to
# a non-text file and that the restore can then be used to
@@ -54,6 +64,58 @@ my %pgdump_runs = (
"$tempdir/binary_upgrade.dump",
],
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_custom => {
+ test_key => 'compression',
+ compile_option => 'gzip',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=1', "--file=$tempdir/compression_gzip_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_gzip_custom.sql",
+ "$tempdir/compression_gzip_custom.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_gzip_dir => {
+ test_key => 'compression',
+ compile_option => 'gzip',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=1',
+ "--file=$tempdir/compression_gzip_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'GZIP_PROGRAM'},
+ args => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_gzip_dir.sql",
+ "$tempdir/compression_gzip_dir",
+ ],
+ },
+
+ compression_gzip_plain => {
+ test_key => 'compression',
+ compile_option => 'gzip',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '-Z1',
+ "--file=$tempdir/compression_gzip_plain.sql.gz", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'GZIP_PROGRAM'},
+ args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
+ },
+ },
clean => {
dump_cmd => [
'pg_dump',
@@ -424,6 +486,7 @@ my %full_runs = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3098,6 +3161,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_test_table => 1,
@@ -3171,6 +3235,7 @@ my %tests = (
binary_upgrade => 1,
clean => 1,
clean_if_exists => 1,
+ compression => 1,
createdb => 1,
defaults => 1,
exclude_dump_test_schema => 1,
@@ -3833,8 +3898,9 @@ if ($collation_check_stderr !~ /ERROR: /)
$collation_support = 1;
}
-# Determine whether build supports LZ4.
-my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
+# Determine whether build supports LZ4 and gzip.
+my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
# Create additional databases for mutations of schema public
$node->psql('postgres', 'create database regress_pg_dump_test;');
@@ -3947,9 +4013,32 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
my $run_db = 'postgres';
+ # Skip command-level tests for gzip if there is no support for it.
+ if ( defined($pgdump_runs{$run}->{compile_option})
+ && $pgdump_runs{$run}->{compile_option} eq 'gzip'
+ && !$supports_gzip)
+ {
+ note "$run: skipped due to no gzip support";
+ next;
+ }
+
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
+ if ($pgdump_runs{$run}->{compress_cmd})
+ {
+ my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd};
+ my $compress_program = $compress_cmd->{program};
+
+ # Skip the rest of the test if the compression program is
+ # not defined.
+ next if (!defined($compress_program) || $compress_program eq '');
+
+ my @full_compress_cmd =
+ ($compress_cmd->{program}, @{ $compress_cmd->{args} });
+ command_ok(\@full_compress_cmd, "$run: compression commands");
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.35.1
[application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-04-01 15:06 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-04-05 01:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-04-05 10:55 ` Michael Paquier <[email protected]>
1 sibling, 0 replies; 67+ messages in thread
From: Michael Paquier @ 2022-04-05 10:55 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Tue, Apr 05, 2022 at 07:13:35AM +0000, [email protected] wrote:
> Thank you. I agree with the attached and I will carry it forward to the
> rest of the patchset.
No need to carry it forward anymore, I think ;)
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-04-01 15:06 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-04-05 01:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2025-10-12 17:24 ` Tom Lane <[email protected]>
2025-10-14 05:40 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
1 sibling, 1 reply; 67+ messages in thread
From: Tom Lane @ 2025-10-12 17:24 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: [email protected]; Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
[ blast-from-the-past department ]
Michael Paquier <[email protected]> writes:
> At the end I am finishing with the attached. I also saw an overlap
> with the addition of --jobs for the directory format vs not using the
> option, so I have removed the case where --jobs was not used in the
> directory format.
(This patch became commit 98fe74218.)
I am wondering if you remember why this bit:
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'GZIP_PROGRAM'},
+ args => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
+ },
was set up to manually compress blobs.toc but not the main TOC in the
toc.dat file. It turns out that Gzip_read is broken for the case
of a zero-length read request [1], but we never reach that case
unless toc.dat is compressed. We don't cover the getc_func member
of the compression stream API, either.
I thought for a bit about proposing that we compress toc.dat but not
blobs.toc, but that loses coverage in another way: the gets_func API
turns out to be used only while reading blobs.toc. So we need to
compress both files manually if we want full coverage.
I think this change won't lose coverage, because there are other tests
in 002_pg_dump.pl that exercise directory format without extra
compression of anything.
Thoughts?
regards, tom lane
[1] https://www.postgresql.org/message-id/3686.1760232320%40sss.pgh.pa.us
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-04-01 15:06 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-04-05 01:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2025-10-12 17:24 ` Re: Add LZ4 compression in pg_dump Tom Lane <[email protected]>
@ 2025-10-14 05:40 ` Michael Paquier <[email protected]>
0 siblings, 0 replies; 67+ messages in thread
From: Michael Paquier @ 2025-10-14 05:40 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: [email protected]; Justin Pryzby <[email protected]>; Rachel Heaton <[email protected]>; Greg Stark <[email protected]>; [email protected]
On Sun, Oct 12, 2025 at 01:24:37PM -0400, Tom Lane wrote:
> Michael Paquier <[email protected]> writes:
>> At the end I am finishing with the attached. I also saw an overlap
>> with the addition of --jobs for the directory format vs not using the
>> option, so I have removed the case where --jobs was not used in the
>> directory format.
>
> (This patch became commit 98fe74218.)
>
> I am wondering if you remember why this bit:
>
> + # Give coverage for manually compressed blob.toc files during
> + # restore.
> + compress_cmd => {
> + program => $ENV{'GZIP_PROGRAM'},
> + args => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
> + },
>
> was set up to manually compress blobs.toc but not the main TOC in the
> toc.dat file. It turns out that Gzip_read is broken for the case
> of a zero-length read request [1], but we never reach that case
> unless toc.dat is compressed. We don't cover the getc_func member
> of the compression stream API, either.
Using extra commands to emulate the manual compressions of the dump
elements is an idea that comes originally from Georgios, and we
included in the first version of the patch posted on the original
thread:
https://www.postgresql.org/message-id/faUNEOpts9vunEaLnmxmG-DldLSg_ql137OC3JYDmgrOMHm1RvvWY2IdBkv_CR...
If I recall correctly, it's something that the two of us have
discussed offline because Georgios had sent the patch to the lists.
He has argued about these additions in favor of coverage, because he
had noticed a bunch of edge cases we never reached with only the tests
in core while coding compression support. Regarding toc.dat
specifically, I think that we have just failed to consider it as a
"valid" case, as far as I can see. So I see no reason to not adding
coverage where toc.dat is manuall compressed, and you are giving a
good reason to add this part in the test.
> I thought for a bit about proposing that we compress toc.dat but not
> blobs.toc, but that loses coverage in another way: the gets_func API
> turns out to be used only while reading blobs.toc. So we need to
> compress both files manually if we want full coverage.
>
> I think this change won't lose coverage, because there are other tests
> in 002_pg_dump.pl that exercise directory format without extra
> compression of anything.
>
> Thoughts?
It looks like it should be OK to just add a "manual" compression of
toc.dat in compression_gzip_dir. That sounds OK to me here. As far
as I can see on HEAD at 1c05fe11abb6, the problem has been fixed in
1f8062dd9668, without a test added for toc.dat. Having coverage would
be nice. I would also expand this concept to compression_zstd_dir and
compression_lz4_dir, but I guess that you are implying that already to
provide coverage for all the compression methods supported by pg_dump?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
@ 2022-03-26 16:21 ` Justin Pryzby <[email protected]>
2022-03-26 18:33 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-26 23:37 ` Re: Add LZ4 compression in pg_dump Daniel Gustafsson <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
1 sibling, 4 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-03-26 16:21 UTC (permalink / raw)
To: Georgios <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>
LZ4F_HEADER_SIZE_MAX isn't defined in old LZ4.
I ran into that on an ubuntu LTS, so I don't think it's so old that it
shouldn't be handled more gracefully. LZ4 should either have an explicit
version check, or else shouldn't depend on that feature (or should define a
safe fallback version if the library header doesn't define it).
https://packages.ubuntu.com/liblz4-1
0003: typo: of legacy => or legacy
There are a large number of ifdefs being added here - it'd be nice to minimize
that. basebackup was organized to use separate files, which is one way.
$ git grep -c 'ifdef .*LZ4' src/bin/pg_dump/compress_io.c
src/bin/pg_dump/compress_io.c:19
In last year's CF entry, I had made a union within CompressorState. LZ4
doesn't need z_streamp (and ztsd will need ZSTD_outBuffer, ZSTD_inBuffer,
ZSTD_CStream).
0002: I wonder if you're able to re-use any of the basebackup parsing stuff
from commit ffd53659c. You're passing both the compression method *and* level.
I think there should be a structure which includes both. In the future, that
can also handle additional options. I hope to re-use these same things for
wal_compression=method:level.
You renamed this:
|- COMPR_ALG_LIBZ
|-} CompressionAlgorithm;
|+ COMPRESSION_GZIP,
|+} CompressionMethod;
..But I don't think that's an improvement. If you were to change it, it should
say something like PGDUMP_COMPRESS_ZLIB, since there are other compression
structs and typedefs. zlib is not idential to gzip, which uses a different
header, so in WriteDataToArchive(), LIBZ is correct, and GZIP is incorrect.
The cf* changes in pg_backup_archiver could be split out into a separate
commit. It's strictly a code simplification - not just preparation for more
compression algorithms. The commit message should "See also:
bf9aa490db24b2334b3595ee33653bf2fe39208c".
The changes in 0002 for cfopen_write seem insufficient:
|+ if (compressionMethod == COMPRESSION_NONE)
|+ fp = cfopen(path, mode, compressionMethod, 0);
| else
| {
| #ifdef HAVE_LIBZ
| char *fname;
|
| fname = psprintf("%s.gz", path);
|- fp = cfopen(fname, mode, compression);
|+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
| free_keep_errno(fname);
| #else
The only difference between the LIBZ and uncompressed case is the file
extension, and it'll be the only difference with LZ4 too. So I suggest to
first handle the file extension, and the rest of the code path is not
conditional on the compression method. I don't think cfopen_write even needs
HAVE_LIBZ - can't you handle that in cfopen_internal() ?
This patch rejects -Z0, which ought to be accepted:
./src/bin/pg_dump/pg_dump -h /tmp regression -Fc -Z0 |wc
pg_dump: error: can only specify -Z/--compress [LEVEL] when method is set
Your 0003 patch shouldn't reference LZ4:
+#ifndef HAVE_LIBLZ4
+ if (*compressionMethod == COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
The 0004 patch renames zlibOutSize to outsize - I think the patch series should
be constructed such as to minimize the size of the method-specific patches. I
say this anticipating also adding support for zstd. The preliminary patches
should have all the boring stuff. It would help for reviewing to keep the
patches split up, or to enumerate all the boring things that are being renamed
(like change OutputContext to cfp, rename zlibOutSize, ...).
0004: The include should use <lz4.h> and not "lz4.h"
freebsd/cfbot is failing.
I suggested off-list to add an 0099 patch to change LZ4 to the default, to
exercise it more on CI.
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-26 18:33 ` Justin Pryzby <[email protected]>
3 siblings, 0 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-03-26 18:33 UTC (permalink / raw)
To: Georgios <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>
On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
> You're passing both the compression method *and* level. I think there should
> be a structure which includes both. In the future, that can also handle
> additional options.
I'm not sure if there's anything worth saving, but I did that last year with
0003-Support-multiple-compression-algs-levels-opts.patch
I sent a rebased copy off-list.
https://www.postgresql.org/message-id/flat/[email protected]#ca1b9f9d3552c87fa8747...
| fatal("not built with LZ4 support");
| fatal("not built with lz4 support");
Please use consistent capitalization of "lz4" - then the compiler can optimize
away duplicate strings.
> 0004: The include should use <lz4.h> and not "lz4.h"
Also, use USE_LZ4 rather than HAVE_LIBLZ4, per 75eae0908.
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-26 23:37 ` Daniel Gustafsson <[email protected]>
2022-03-26 23:51 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
3 siblings, 1 reply; 67+ messages in thread
From: Daniel Gustafsson @ 2022-03-26 23:37 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Georgios <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
> On 26 Mar 2022, at 17:21, Justin Pryzby <[email protected]> wrote:
> I suggested off-list to add an 0099 patch to change LZ4 to the default, to
> exercise it more on CI.
No need to change the defaults in autoconf for that. The CFBot uses the cirrus
file in the tree so changing what the job includes can be easily done (assuming
the CFBot hasn't changed this recently which I think it hasn't). I used that
trick in the NSS patchset to add a completely new job for --with-ssl=nss beside
the --with-ssl=openssl job.
--
Daniel Gustafsson https://vmware.com/
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-26 23:37 ` Re: Add LZ4 compression in pg_dump Daniel Gustafsson <[email protected]>
@ 2022-03-26 23:51 ` Justin Pryzby <[email protected]>
2022-03-27 19:55 ` Re: Add LZ4 compression in pg_dump Daniel Gustafsson <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-03-26 23:51 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Georgios <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Sun, Mar 27, 2022 at 12:37:27AM +0100, Daniel Gustafsson wrote:
> > On 26 Mar 2022, at 17:21, Justin Pryzby <[email protected]> wrote:
>
> > I suggested off-list to add an 0099 patch to change LZ4 to the default, to
> > exercise it more on CI.
>
> No need to change the defaults in autoconf for that. The CFBot uses the cirrus
> file in the tree so changing what the job includes can be easily done (assuming
> the CFBot hasn't changed this recently which I think it hasn't). I used that
> trick in the NSS patchset to add a completely new job for --with-ssl=nss beside
> the --with-ssl=openssl job.
I think you misunderstood - I'm suggesting not only to use with-lz4 (which was
always true since 93d973494), but to change pg_dump -Fc and -Fd to use LZ4 by
default (the same as I suggested for toast_compression, wal_compression, and
again in last year's patch to add zstd compression to pg_dump, for which
postgres was not ready).
@@ -781,6 +807,11 @@ main(int argc, char **argv)
compress.alg = COMPR_ALG_LIBZ;
compress.level = Z_DEFAULT_COMPRESSION;
#endif
+
+#ifdef USE_ZSTD
+ compress.alg = COMPR_ALG_ZSTD; // Set default for testing purposes
+ compress.level = ZSTD_CLEVEL_DEFAULT;
+#endif
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-26 23:37 ` Re: Add LZ4 compression in pg_dump Daniel Gustafsson <[email protected]>
2022-03-26 23:51 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-27 19:55 ` Daniel Gustafsson <[email protected]>
0 siblings, 0 replies; 67+ messages in thread
From: Daniel Gustafsson @ 2022-03-27 19:55 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
> On 27 Mar 2022, at 00:51, Justin Pryzby <[email protected]> wrote:
>
> On Sun, Mar 27, 2022 at 12:37:27AM +0100, Daniel Gustafsson wrote:
>>> On 26 Mar 2022, at 17:21, Justin Pryzby <[email protected]> wrote:
>>
>>> I suggested off-list to add an 0099 patch to change LZ4 to the default, to
>>> exercise it more on CI.
>>
>> No need to change the defaults in autoconf for that. The CFBot uses the cirrus
>> file in the tree so changing what the job includes can be easily done (assuming
>> the CFBot hasn't changed this recently which I think it hasn't). I used that
>> trick in the NSS patchset to add a completely new job for --with-ssl=nss beside
>> the --with-ssl=openssl job.
>
> I think you misunderstood - I'm suggesting not only to use with-lz4 (which was
> always true since 93d973494), but to change pg_dump -Fc and -Fd to use LZ4 by
> default (the same as I suggested for toast_compression, wal_compression, and
> again in last year's patch to add zstd compression to pg_dump, for which
> postgres was not ready).
Right, I clearly misunderstood, thanks for the clarification.
--
Daniel Gustafsson https://vmware.com/
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-27 14:13 ` Robert Haas <[email protected]>
2022-03-27 16:06 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
3 siblings, 1 reply; 67+ messages in thread
From: Robert Haas @ 2022-03-27 14:13 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
On Sat, Mar 26, 2022 at 12:22 PM Justin Pryzby <[email protected]> wrote:
> 0002: I wonder if you're able to re-use any of the basebackup parsing stuff
> from commit ffd53659c. You're passing both the compression method *and* level.
> I think there should be a structure which includes both. In the future, that
> can also handle additional options. I hope to re-use these same things for
> wal_compression=method:level.
Yeah, we should really try to use that infrastructure instead of
inventing a bunch of different ways to do it. It might require some
renaming here and there, and I'm not sure whether we really want to
try to rush all this into the current release, but I think we should
find a way to get it done.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
@ 2022-03-27 16:06 ` Justin Pryzby <[email protected]>
2022-03-28 12:36 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-03-27 16:06 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Georgios <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Sun, Mar 27, 2022 at 10:13:00AM -0400, Robert Haas wrote:
> On Sat, Mar 26, 2022 at 12:22 PM Justin Pryzby <[email protected]> wrote:
> > 0002: I wonder if you're able to re-use any of the basebackup parsing stuff
> > from commit ffd53659c. You're passing both the compression method *and* level.
> > I think there should be a structure which includes both. In the future, that
> > can also handle additional options. I hope to re-use these same things for
> > wal_compression=method:level.
>
> Yeah, we should really try to use that infrastructure instead of
> inventing a bunch of different ways to do it. It might require some
> renaming here and there, and I'm not sure whether we really want to
> try to rush all this into the current release, but I think we should
> find a way to get it done.
It seems like something a whole lot like parse_compress_options() should be in
common/. Nobody wants to write it again, and I couldn't convince myself to
copy it when I looked at using it for wal_compression.
Maybe it should take an argument which specifies the default algorithm to use
for input of a numeric "level". And reject such input if not specified, since
wal_compression has never taken a "level", so it's not useful or desirable to
have that default to some new algorithm.
I could write this down if you want, although I'm not sure how/if you intend
other people to use bc_algorithm and bc_algorithm. I don't think it's
important to do for v15, but it seems like it could be done after featue
freeze. pg_dump+lz4 is targetting v16, although there's a cleanup patch that
could also go in before branching.
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-27 16:06 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-03-28 12:36 ` Robert Haas <[email protected]>
2022-03-29 05:03 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Robert Haas @ 2022-03-28 12:36 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
On Sun, Mar 27, 2022 at 12:06 PM Justin Pryzby <[email protected]> wrote:
> Maybe it should take an argument which specifies the default algorithm to use
> for input of a numeric "level". And reject such input if not specified, since
> wal_compression has never taken a "level", so it's not useful or desirable to
> have that default to some new algorithm.
That sounds odd to me. Wouldn't it be rather confusing if a bare
integer meant gzip for one case and lz4 for another?
> I could write this down if you want, although I'm not sure how/if you intend
> other people to use bc_algorithm and bc_algorithm. I don't think it's
> important to do for v15, but it seems like it could be done after featue
> freeze. pg_dump+lz4 is targetting v16, although there's a cleanup patch that
> could also go in before branching.
Well, I think the first thing we should do is get rid of enum
WalCompressionMethod and use enum WalCompression instead. They've got
the same elements and very similar names, but the WalCompressionMethod
ones just have names like COMPRESSION_NONE, which is too generic,
whereas WalCompressionMethod uses WAL_COMPRESSION_NONE, which is
better. Then I think we should also rename the COMPR_ALG_* constants
in pg_dump.h to names like DUMP_COMPRESSION_*. Once we do that we've
got rid of all the unprefixed things that purport to be a list of
compression algorithms.
Then, if people are willing to adopt the syntax that the
backup_compression.c/h stuff supports as a project standard (+1 from
me) we can go the other way and rename that stuff to be more generic,
taking backup out of the name.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-27 16:06 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-28 12:36 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
@ 2022-03-29 05:03 ` Michael Paquier <[email protected]>
2022-03-29 13:14 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-03-29 05:03 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
On Mon, Mar 28, 2022 at 08:36:15AM -0400, Robert Haas wrote:
> Well, I think the first thing we should do is get rid of enum
> WalCompressionMethod and use enum WalCompression instead. They've got
> the same elements and very similar names, but the WalCompressionMethod
> ones just have names like COMPRESSION_NONE, which is too generic,
> whereas WalCompressionMethod uses WAL_COMPRESSION_NONE, which is
> better. Then I think we should also rename the COMPR_ALG_* constants
> in pg_dump.h to names like DUMP_COMPRESSION_*. Once we do that we've
> got rid of all the unprefixed things that purport to be a list of
> compression algorithms.
Yes, having a centralized enum for the compression method would make
sense, along with the routines to parse and get the compression method
names. At least that would be one step towards more unity in
src/common/.
> Then, if people are willing to adopt the syntax that the
> backup_compression.c/h stuff supports as a project standard (+1 from
> me) we can go the other way and rename that stuff to be more generic,
> taking backup out of the name.
I am not sure about the specification part which is only used by base
backups that has no client-server requirements, so option values would
still require their own grammar.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-27 16:06 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-28 12:36 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-29 05:03 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-03-29 13:14 ` Robert Haas <[email protected]>
2022-03-30 06:36 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Robert Haas @ 2022-03-29 13:14 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
On Tue, Mar 29, 2022 at 1:03 AM Michael Paquier <[email protected]> wrote:
> > Then, if people are willing to adopt the syntax that the
> > backup_compression.c/h stuff supports as a project standard (+1 from
> > me) we can go the other way and rename that stuff to be more generic,
> > taking backup out of the name.
>
> I am not sure about the specification part which is only used by base
> backups that has no client-server requirements, so option values would
> still require their own grammar.
I don't know what you mean by this. I think the specification stuff
could be reused in a lot of places. If you can ask for a base backup
with zstd:level=3,long=1,fancystuff=yes or whatever we end up with,
why not enable exactly the same for every other place that uses
compression? I don't know what "client-server requirements" is or what
that has to do with this.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-27 16:06 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-28 12:36 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-29 05:03 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 13:14 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
@ 2022-03-30 06:36 ` Michael Paquier <[email protected]>
0 siblings, 0 replies; 67+ messages in thread
From: Michael Paquier @ 2022-03-30 06:36 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Georgios <[email protected]>; PostgreSQL Hackers <[email protected]>; Rachel Heaton <[email protected]>
On Tue, Mar 29, 2022 at 09:14:03AM -0400, Robert Haas wrote:
> I don't know what you mean by this. I think the specification stuff
> could be reused in a lot of places. If you can ask for a base backup
> with zstd:level=3,long=1,fancystuff=yes or whatever we end up with,
> why not enable exactly the same for every other place that uses
> compression? I don't know what "client-server requirements" is or what
> that has to do with this.
Oh. I think that I got confused here. I saw the backup component in
the file name and this has been associated with the client/server
choice that can be done in the options of pg_basebackup. But
parse_bc_specification() does not include any knowledge about that:
pg_basebackup does this job in parse_compress_options(). I agree that
it looks possible to reuse that stuff in more places than just base
backups.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-06-26 15:55 ` Justin Pryzby <[email protected]>
2022-06-28 08:14 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
3 siblings, 2 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-06-26 15:55 UTC (permalink / raw)
To: Georgios <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>
Hi,
Will you be able to send a rebased patch for the next CF ?
If you update for the review comments I sent in March, I'll plan to do another
round of review.
On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
> LZ4F_HEADER_SIZE_MAX isn't defined in old LZ4.
>
> I ran into that on an ubuntu LTS, so I don't think it's so old that it
> shouldn't be handled more gracefully. LZ4 should either have an explicit
> version check, or else shouldn't depend on that feature (or should define a
> safe fallback version if the library header doesn't define it).
>
> https://packages.ubuntu.com/liblz4-1
>
> 0003: typo: of legacy => or legacy
>
> There are a large number of ifdefs being added here - it'd be nice to minimize
> that. basebackup was organized to use separate files, which is one way.
>
> $ git grep -c 'ifdef .*LZ4' src/bin/pg_dump/compress_io.c
> src/bin/pg_dump/compress_io.c:19
>
> In last year's CF entry, I had made a union within CompressorState. LZ4
> doesn't need z_streamp (and ztsd will need ZSTD_outBuffer, ZSTD_inBuffer,
> ZSTD_CStream).
>
> 0002: I wonder if you're able to re-use any of the basebackup parsing stuff
> from commit ffd53659c. You're passing both the compression method *and* level.
> I think there should be a structure which includes both. In the future, that
> can also handle additional options. I hope to re-use these same things for
> wal_compression=method:level.
>
> You renamed this:
>
> |- COMPR_ALG_LIBZ
> |-} CompressionAlgorithm;
> |+ COMPRESSION_GZIP,
> |+} CompressionMethod;
>
> ..But I don't think that's an improvement. If you were to change it, it should
> say something like PGDUMP_COMPRESS_ZLIB, since there are other compression
> structs and typedefs. zlib is not idential to gzip, which uses a different
> header, so in WriteDataToArchive(), LIBZ is correct, and GZIP is incorrect.
>
> The cf* changes in pg_backup_archiver could be split out into a separate
> commit. It's strictly a code simplification - not just preparation for more
> compression algorithms. The commit message should "See also:
> bf9aa490db24b2334b3595ee33653bf2fe39208c".
>
> The changes in 0002 for cfopen_write seem insufficient:
> |+ if (compressionMethod == COMPRESSION_NONE)
> |+ fp = cfopen(path, mode, compressionMethod, 0);
> | else
> | {
> | #ifdef HAVE_LIBZ
> | char *fname;
> |
> | fname = psprintf("%s.gz", path);
> |- fp = cfopen(fname, mode, compression);
> |+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
> | free_keep_errno(fname);
> | #else
>
> The only difference between the LIBZ and uncompressed case is the file
> extension, and it'll be the only difference with LZ4 too. So I suggest to
> first handle the file extension, and the rest of the code path is not
> conditional on the compression method. I don't think cfopen_write even needs
> HAVE_LIBZ - can't you handle that in cfopen_internal() ?
>
> This patch rejects -Z0, which ought to be accepted:
> ./src/bin/pg_dump/pg_dump -h /tmp regression -Fc -Z0 |wc
> pg_dump: error: can only specify -Z/--compress [LEVEL] when method is set
>
> Your 0003 patch shouldn't reference LZ4:
> +#ifndef HAVE_LIBLZ4
> + if (*compressionMethod == COMPRESSION_LZ4)
> + supports_compression = false;
> +#endif
>
> The 0004 patch renames zlibOutSize to outsize - I think the patch series should
> be constructed such as to minimize the size of the method-specific patches. I
> say this anticipating also adding support for zstd. The preliminary patches
> should have all the boring stuff. It would help for reviewing to keep the
> patches split up, or to enumerate all the boring things that are being renamed
> (like change OutputContext to cfp, rename zlibOutSize, ...).
>
> 0004: The include should use <lz4.h> and not "lz4.h"
>
> freebsd/cfbot is failing.
>
> I suggested off-list to add an 0099 patch to change LZ4 to the default, to
> exercise it more on CI.
On Sat, Mar 26, 2022 at 01:33:36PM -0500, Justin Pryzby wrote:
> On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
> > You're passing both the compression method *and* level. I think there should
> > be a structure which includes both. In the future, that can also handle
> > additional options.
>
> I'm not sure if there's anything worth saving, but I did that last year with
> 0003-Support-multiple-compression-algs-levels-opts.patch
> I sent a rebased copy off-list.
> https://www.postgresql.org/message-id/flat/[email protected]#ca1b9f9d3552c87fa8747...
>
> | fatal("not built with LZ4 support");
> | fatal("not built with lz4 support");
>
> Please use consistent capitalization of "lz4" - then the compiler can optimize
> away duplicate strings.
>
> > 0004: The include should use <lz4.h> and not "lz4.h"
>
> Also, use USE_LZ4 rather than HAVE_LIBLZ4, per 75eae0908.
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-06-28 08:14 ` [email protected]
1 sibling, 0 replies; 67+ messages in thread
From: [email protected] @ 2022-06-28 08:14 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Sunday, June 26th, 2022 at 5:55 PM, Justin Pryzby <[email protected]> wrote:
>
>
> Hi,
>
> Will you be able to send a rebased patch for the next CF ?
Thank you for taking an interest in the PR. The plan is indeed to sent
a new version.
> If you update for the review comments I sent in March, I'll plan to do another
> round of review.
Thank you.
>
> On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
>
> > LZ4F_HEADER_SIZE_MAX isn't defined in old LZ4.
> >
> > I ran into that on an ubuntu LTS, so I don't think it's so old that it
> > shouldn't be handled more gracefully. LZ4 should either have an explicit
> > version check, or else shouldn't depend on that feature (or should define a
> > safe fallback version if the library header doesn't define it).
> >
> > https://packages.ubuntu.com/liblz4-1
> >
> > 0003: typo: of legacy => or legacy
> >
> > There are a large number of ifdefs being added here - it'd be nice to minimize
> > that. basebackup was organized to use separate files, which is one way.
> >
> > $ git grep -c 'ifdef .*LZ4' src/bin/pg_dump/compress_io.c
> > src/bin/pg_dump/compress_io.c:19
> >
> > In last year's CF entry, I had made a union within CompressorState. LZ4
> > doesn't need z_streamp (and ztsd will need ZSTD_outBuffer, ZSTD_inBuffer,
> > ZSTD_CStream).
> >
> > 0002: I wonder if you're able to re-use any of the basebackup parsing stuff
> > from commit ffd53659c. You're passing both the compression method and level.
> > I think there should be a structure which includes both. In the future, that
> > can also handle additional options. I hope to re-use these same things for
> > wal_compression=method:level.
> >
> > You renamed this:
> >
> > |- COMPR_ALG_LIBZ
> > |-} CompressionAlgorithm;
> > |+ COMPRESSION_GZIP,
> > |+} CompressionMethod;
> >
> > ..But I don't think that's an improvement. If you were to change it, it should
> > say something like PGDUMP_COMPRESS_ZLIB, since there are other compression
> > structs and typedefs. zlib is not idential to gzip, which uses a different
> > header, so in WriteDataToArchive(), LIBZ is correct, and GZIP is incorrect.
> >
> > The cf* changes in pg_backup_archiver could be split out into a separate
> > commit. It's strictly a code simplification - not just preparation for more
> > compression algorithms. The commit message should "See also:
> > bf9aa490db24b2334b3595ee33653bf2fe39208c".
> >
> > The changes in 0002 for cfopen_write seem insufficient:
> > |+ if (compressionMethod == COMPRESSION_NONE)
> > |+ fp = cfopen(path, mode, compressionMethod, 0);
> > | else
> > | {
> > | #ifdef HAVE_LIBZ
> > | char *fname;
> > |
> > | fname = psprintf("%s.gz", path);
> > |- fp = cfopen(fname, mode, compression);
> > |+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
> > | free_keep_errno(fname);
> > | #else
> >
> > The only difference between the LIBZ and uncompressed case is the file
> > extension, and it'll be the only difference with LZ4 too. So I suggest to
> > first handle the file extension, and the rest of the code path is not
> > conditional on the compression method. I don't think cfopen_write even needs
> > HAVE_LIBZ - can't you handle that in cfopen_internal() ?
> >
> > This patch rejects -Z0, which ought to be accepted:
> > ./src/bin/pg_dump/pg_dump -h /tmp regression -Fc -Z0 |wc
> > pg_dump: error: can only specify -Z/--compress [LEVEL] when method is set
> >
> > Your 0003 patch shouldn't reference LZ4:
> > +#ifndef HAVE_LIBLZ4
> > + if (*compressionMethod == COMPRESSION_LZ4)
> > + supports_compression = false;
> > +#endif
> >
> > The 0004 patch renames zlibOutSize to outsize - I think the patch series should
> > be constructed such as to minimize the size of the method-specific patches. I
> > say this anticipating also adding support for zstd. The preliminary patches
> > should have all the boring stuff. It would help for reviewing to keep the
> > patches split up, or to enumerate all the boring things that are being renamed
> > (like change OutputContext to cfp, rename zlibOutSize, ...).
> >
> > 0004: The include should use <lz4.h> and not "lz4.h"
> >
> > freebsd/cfbot is failing.
> >
> > I suggested off-list to add an 0099 patch to change LZ4 to the default, to
> > exercise it more on CI.
>
>
> On Sat, Mar 26, 2022 at 01:33:36PM -0500, Justin Pryzby wrote:
>
> > On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
> >
> > > You're passing both the compression method and level. I think there should
> > > be a structure which includes both. In the future, that can also handle
> > > additional options.
> >
> > I'm not sure if there's anything worth saving, but I did that last year with
> > 0003-Support-multiple-compression-algs-levels-opts.patch
> > I sent a rebased copy off-list.
> > https://www.postgresql.org/message-id/flat/[email protected]#ca1b9f9d3552c87fa8747...
> >
> > | fatal("not built with LZ4 support");
> > | fatal("not built with lz4 support");
> >
> > Please use consistent capitalization of "lz4" - then the compiler can optimize
> > away duplicate strings.
> >
> > > 0004: The include should use <lz4.h> and not "lz4.h"
> >
> > Also, use USE_LZ4 rather than HAVE_LIBLZ4, per 75eae0908.
>
>
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-07-05 13:22 ` [email protected]
2022-07-05 15:13 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-21 05:04 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
1 sibling, 3 replies; 67+ messages in thread
From: [email protected] @ 2022-07-05 13:22 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>; Michael Paquier <[email protected]>
------- Original Message -------
On Sunday, June 26th, 2022 at 5:55 PM, Justin Pryzby <[email protected]> wrote:
>
> Hi,
>
> Will you be able to send a rebased patch for the next CF ?
Please find a rebased and heavily refactored patchset. Since parts of this
patchset were already committed, I restarted numbering. I am not certain if
this is the preferred way. This makes alignment with previous comments a bit
harder
> If you update for the review comments I sent in March, I'll plan to do another
> round of review.
I have updated for "some" of the comments. This is not an unwillingness to
incorporate those specific comments. Simply this patchset had started to divert
heavily already based on comments from Mr. Paquier who had already requested for
the APIs to be refactored to use function pointers. This is happening in 0002 of
the patchset. 0001 of the patchset is using the new compression.h under common.
This patchset should be considered a late draft, as commentary, documentation,
and some finer details are not yet finalized; because I am expecting the proposed
refactor to receive a wealth of comments. It would be helpful to understand if
the proposed direction is something worth to be worked upon, before moving to the
finer details.
For what is worth, I am the sole author of the current patchset.
Cheers,
//Georgios
> On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
>
> > LZ4F_HEADER_SIZE_MAX isn't defined in old LZ4.
> >
> > I ran into that on an ubuntu LTS, so I don't think it's so old that it
> > shouldn't be handled more gracefully. LZ4 should either have an explicit
> > version check, or else shouldn't depend on that feature (or should define a
> > safe fallback version if the library header doesn't define it).
> >
> > https://packages.ubuntu.com/liblz4-1
> >
> > 0003: typo: of legacy => or legacy
> >
> > There are a large number of ifdefs being added here - it'd be nice to minimize
> > that. basebackup was organized to use separate files, which is one way.
> >
> > $ git grep -c 'ifdef .*LZ4' src/bin/pg_dump/compress_io.c
> > src/bin/pg_dump/compress_io.c:19
> >
> > In last year's CF entry, I had made a union within CompressorState. LZ4
> > doesn't need z_streamp (and ztsd will need ZSTD_outBuffer, ZSTD_inBuffer,
> > ZSTD_CStream).
> >
> > 0002: I wonder if you're able to re-use any of the basebackup parsing stuff
> > from commit ffd53659c. You're passing both the compression method and level.
> > I think there should be a structure which includes both. In the future, that
> > can also handle additional options. I hope to re-use these same things for
> > wal_compression=method:level.
> >
> > You renamed this:
> >
> > |- COMPR_ALG_LIBZ
> > |-} CompressionAlgorithm;
> > |+ COMPRESSION_GZIP,
> > |+} CompressionMethod;
> >
> > ..But I don't think that's an improvement. If you were to change it, it should
> > say something like PGDUMP_COMPRESS_ZLIB, since there are other compression
> > structs and typedefs. zlib is not idential to gzip, which uses a different
> > header, so in WriteDataToArchive(), LIBZ is correct, and GZIP is incorrect.
> >
> > The cf* changes in pg_backup_archiver could be split out into a separate
> > commit. It's strictly a code simplification - not just preparation for more
> > compression algorithms. The commit message should "See also:
> > bf9aa490db24b2334b3595ee33653bf2fe39208c".
> >
> > The changes in 0002 for cfopen_write seem insufficient:
> > |+ if (compressionMethod == COMPRESSION_NONE)
> > |+ fp = cfopen(path, mode, compressionMethod, 0);
> > | else
> > | {
> > | #ifdef HAVE_LIBZ
> > | char *fname;
> > |
> > | fname = psprintf("%s.gz", path);
> > |- fp = cfopen(fname, mode, compression);
> > |+ fp = cfopen(fname, mode, compressionMethod, compressionLevel);
> > | free_keep_errno(fname);
> > | #else
> >
> > The only difference between the LIBZ and uncompressed case is the file
> > extension, and it'll be the only difference with LZ4 too. So I suggest to
> > first handle the file extension, and the rest of the code path is not
> > conditional on the compression method. I don't think cfopen_write even needs
> > HAVE_LIBZ - can't you handle that in cfopen_internal() ?
> >
> > This patch rejects -Z0, which ought to be accepted:
> > ./src/bin/pg_dump/pg_dump -h /tmp regression -Fc -Z0 |wc
> > pg_dump: error: can only specify -Z/--compress [LEVEL] when method is set
> >
> > Your 0003 patch shouldn't reference LZ4:
> > +#ifndef HAVE_LIBLZ4
> > + if (*compressionMethod == COMPRESSION_LZ4)
> > + supports_compression = false;
> > +#endif
> >
> > The 0004 patch renames zlibOutSize to outsize - I think the patch series should
> > be constructed such as to minimize the size of the method-specific patches. I
> > say this anticipating also adding support for zstd. The preliminary patches
> > should have all the boring stuff. It would help for reviewing to keep the
> > patches split up, or to enumerate all the boring things that are being renamed
> > (like change OutputContext to cfp, rename zlibOutSize, ...).
> >
> > 0004: The include should use <lz4.h> and not "lz4.h"
> >
> > freebsd/cfbot is failing.
> >
> > I suggested off-list to add an 0099 patch to change LZ4 to the default, to
> > exercise it more on CI.
>
>
> On Sat, Mar 26, 2022 at 01:33:36PM -0500, Justin Pryzby wrote:
>
> > On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
> >
> > > You're passing both the compression method and level. I think there should
> > > be a structure which includes both. In the future, that can also handle
> > > additional options.
> >
> > I'm not sure if there's anything worth saving, but I did that last year with
> > 0003-Support-multiple-compression-algs-levels-opts.patch
> > I sent a rebased copy off-list.
> > https://www.postgresql.org/message-id/flat/[email protected]#ca1b9f9d3552c87fa8747...
> >
> > | fatal("not built with LZ4 support");
> > | fatal("not built with lz4 support");
> >
> > Please use consistent capitalization of "lz4" - then the compiler can optimize
> > away duplicate strings.
> >
> > > 0004: The include should use <lz4.h> and not "lz4.h"
> >
> > Also, use USE_LZ4 rather than HAVE_LIBLZ4, per 75eae0908.
>
>
Attachments:
[text/x-patch] v7-0002-Introduce-Compressor-API-in-pg_dump.patch (49.7K, ../../u7dLpRH_6r_rGyqNWHSDx2Wz5g1virAU3-f4XMC2BwiR81Tegqm4Sxtte0YAe77je9KVMa4jVxDNSt3XkYLYpyOoabCid1Bm7BZxwoitcmY=@pm.me/2-v7-0002-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From b8e1b7fe02a898326c8adab9dcc67d9e4c281621 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 5 Jul 2022 12:33:52 +0000
Subject: [PATCH v7 2/3] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 390 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 9 +
src/bin/pg_dump/compress_io.c | 817 ++++++--------------------
src/bin/pg_dump/compress_io.h | 69 ++-
src/bin/pg_dump/pg_backup_archiver.c | 40 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 85 +--
8 files changed, 730 insertions(+), 704 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 2f524b09bf..2d777ec213 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -23,6 +23,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..bc6d1abc77
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,390 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ int compressionLevel;
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, gzipcs->compressionLevel) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+typedef struct GzipData
+{
+ gzFile fp;
+ int compressionLevel;
+} GzipData;
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ size_t ret;
+
+ ret = gzread(gd->fp, ptr, size);
+ if (ret != size && !gzeof(gd->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gd->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzwrite(gd->fp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gd->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gd->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzgets(gd->fp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gd->fp);
+
+ save_errno = errno;
+ free(gd);
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzeof(gd->fp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gd->fp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ char mode_compression[32];
+
+ if (gd->compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, gd->compressionLevel);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gd->fp = gzdopen(dup(fd), mode_compression);
+ else
+ gd->fp = gzopen(path, mode_compression);
+
+ if (gd->fp == NULL)
+ return 1;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ GzipData *gd;
+
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ gd = pg_malloc0(sizeof(GzipData));
+ gd->compressionLevel = compressionLevel;
+
+ CFH->private = gd;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..ab0362c1f3
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, int compressionLevel);
+extern void InitCompressGzip(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 146396172b..1948ee3d57 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
/*----------------------
@@ -61,113 +64,73 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_algorithm compress_algorithm;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+ free(buf);
+}
+
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compress_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compress_algorithm = compress_spec.algorithm;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, compress_spec.level);
-#endif
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
- ReadFunc readF)
-{
switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressorGzip(cs, compress_spec.level);
break;
default:
pg_fatal("invalid compression method");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compress_algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ return cs;
}
/*
@@ -176,243 +139,28 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compress_algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- free(cs);
- break;
-
- default:
- pg_fatal("invalid compression method");
- break;
- }
-}
-
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
- */
-
-static void
-InitCompressorZlib(CompressorState *cs, int level)
-{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
-}
-
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
-{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
-{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
-
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
-
- if (res == Z_STREAM_END)
- break;
- }
-}
-
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
-
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
-{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
-}
-#endif /* HAVE_LIBZ */
-
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
-{
- size_t cnt;
- char *buf;
- size_t buflen;
-
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
-
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
-
- free(buf);
-}
-
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->writeF(AH, data, dLen);
-}
-
-
/*----------------------
* Compressed stream API
*----------------------
*/
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- pg_compress_algorithm compress_algorithm;
- void *fp;
-};
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
+}
/* free() without changing errno; useful in several places below */
static void
@@ -425,392 +173,219 @@ free_keep_errno(void *p)
}
/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
+ * Compression None implementation
*/
-cfp *
-cfopen_read(const char *path, const char *mode)
+static size_t
+_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp;
- pg_compress_specification compress_spec = {0};
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, compress_spec);
- else
-#endif
- {
- compress_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compress_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
-
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
-}
+ if (size == 0)
+ return 0;
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
-{
- cfp *fp;
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
- if (compress_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compress_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ return ret;
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_algorithm compress_algorithm, int compressionLevel)
+static size_t
+_write(const void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
-
- fp->compress_algorithm = compress_algorithm;
-
- switch (compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compressionLevel != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compressionLevel);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
-
- return fp;
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
+static const char *
+_get_error(CompressFileHandle * CFH)
{
- return cfopen_internal(path, -1, mode,
- compress_spec.algorithm,
- compress_spec.level);
+ return strerror(errno);
}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compress_spec)
+static char *
+_gets(char *ptr, int size, CompressFileHandle * CFH)
{
- return cfopen_internal(NULL, fd, mode,
- compress_spec.algorithm,
- compress_spec.level);
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-int
-cfread(void *ptr, int size, cfp *fp)
+static int
+_getc(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret;
- if (size == 0)
- return 0;
-
- switch (fp->compress_algorithm)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
-
- default:
- pg_fatal("invalid compression method");
- break;
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
return ret;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+static int
+_close(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret = 0;
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ CFH->private = NULL;
+
+ if (fp)
+ ret = fclose(fp);
return ret;
}
-int
-cfgetc(cfp *fp)
+static int
+_eof(CompressFileHandle * CFH)
{
- int ret;
+ return feof((FILE *) CFH->private);
+}
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
+static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ Assert(CFH->private == NULL);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- return ret;
+ if (CFH->private == NULL)
+ return 1;
+
+ return 0;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+static int
+_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
{
- char *ret;
+ Assert(CFH->private == NULL);
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ return 0;
+}
- return ret;
+static void
+InitCompressNone(CompressFileHandle * CFH)
+{
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-int
-cfclose(cfp *fp)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compress_spec)
{
- int ret;
+ CompressFileHandle *CFH;
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
- switch (fp->compress_algorithm)
+ switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
+ InitCompressNone(CFH);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressGzip(CFH, compress_spec.level);
break;
default:
pg_fatal("invalid compression method");
break;
}
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
-int
-cfeof(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret;
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compress_spec = {0};
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- return ret;
-}
+ fname = strdup(path);
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
+ if (hasSuffix(fname, ".gz"))
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
+ bool exists;
+
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("not built with zlib support");
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
}
- return strerror(errno);
+ CFH = InitCompressFileHandle(compress_spec);
+ if (CFH->open(fname, -1, mode, CFH))
+ {
+ free_keep_errno(CFH);
+ CFH = NULL;
+ }
+ free_keep_errno(fname);
+
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle * CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index 2b42f030a8..ec13374a52 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -52,34 +52,61 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compress_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle * cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle * CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle * CFH);
+ int (*getc) (CompressFileHandle * CFH);
+ int (*eof) (CompressFileHandle * CFH);
+ int (*close) (CompressFileHandle * CFH);
+ const char *(*get_error) (CompressFileHandle * CFH);
+
+ void *private;
+};
+
-typedef struct cfp cfp;
+extern CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compress_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compress_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle * InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle * CFH);
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index fa84c31ecb..186510c235 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle * SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -354,7 +354,7 @@ RestoreArchive(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1119,7 +1119,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compress_spec;
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1495,6 +1495,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1517,33 +1518,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compress_spec);
- else
- AH->OF = cfopen(filename, mode, compress_spec);
+ CFH = InitCompressFileHandle(compress_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1682,7 +1682,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2171,6 +2175,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2226,7 +2231,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 43ccbe1339..e8f8c4c0d6 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compress_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 17c7130c75..3f35bc9815 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,9 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
+ CompressFileHandle *dataFH; /* currently open data file */
- cfp *blobsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *blobsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +198,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +218,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compress_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +346,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +371,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +386,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +435,7 @@ _LoadBlobs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +443,14 @@ _LoadBlobs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->blobsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the blobs TOC file line-by-line, and process each blob */
- while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char blobfname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +465,11 @@ _LoadBlobs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreBlob(AH, oid);
}
- if (!cfeof(ctx->blobsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +489,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +514,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +527,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +549,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +578,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compress_spec;
char fname[MAXPGPATH];
@@ -584,8 +589,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compress_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +603,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +654,8 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
/* The blob TOC file is never compressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
- if (ctx->blobsTocFH == NULL)
+ ctx->blobsTocFH = InitCompressFileHandle(compress_spec);
+ if (ctx->blobsTocFH->open_write(fname, "ab", ctx->blobsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +672,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,17 +686,18 @@ static void
_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->blobsTocFH;
char buf[50];
int len;
/* Close the BLOB data file itself */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the blob in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->blobsTocFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
pg_fatal("could not write to blobs TOC file");
}
@@ -706,7 +711,7 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close blobs TOC file: %m");
ctx->blobsTocFH = NULL;
}
--
2.34.1
[text/x-patch] v7-0001-Prepare-pg_dump-for-additional-compression-method.patch (50.0K, ../../u7dLpRH_6r_rGyqNWHSDx2Wz5g1virAU3-f4XMC2BwiR81Tegqm4Sxtte0YAe77je9KVMa4jVxDNSt3XkYLYpyOoabCid1Bm7BZxwoitcmY=@pm.me/3-v7-0001-Prepare-pg_dump-for-additional-compression-method.patch)
download | inline diff:
From bf5deb8e3a770e26da876b2268832ab1a07bbb84 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 5 Jul 2022 12:33:28 +0000
Subject: [PATCH v7 1/3] Prepare pg_dump for additional compression methods
This commmit does some of the heavy lifting required for additional compression
methods.
First it is teaching pg_dump.c about the definitions and interfaces found in
common/compression.h. Then it is propagating those throughout the code.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of
additional archive formats. However, pg_backup_archiver.c was not using that
API. This commit teaches pg_backup_archiver.c about cfp and is using it through
out.
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 427 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 35 ++-
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 178 +++++------
src/bin/pg_dump/pg_backup_archiver.h | 37 +--
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 151 +++++++--
src/bin/pg_dump/t/001_basic.pl | 10 +
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
12 files changed, 552 insertions(+), 356 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 5efb442b44..94885d0812 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..146396172b 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_algorithm compress_algorithm;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compress_algorithm = compress_spec.algorithm;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compress_spec.level);
#endif
return cs;
@@ -154,21 +124,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compress_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +152,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compress_algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +176,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compress_algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +406,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_algorithm compress_algorithm;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -452,21 +438,25 @@ cfp *
cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compress_spec = {0};
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compress_spec);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compress_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +469,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compress_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compress_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -509,60 +500,96 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_algorithm compress_algorithm, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compress_algorithm = compress_algorithm;
+
+ switch (compress_algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(path, -1, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(NULL, fd, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +599,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +661,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +694,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compress_algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..2b42f030a8 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -17,16 +17,25 @@
#include "pg_backup_archiver.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#define GZCLOSE(fh) gzclose(fh)
+#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
+#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
+#define GZEOF(fh) gzeof(fh)
+#else
+#define GZCLOSE(fh) fclose(fh)
+#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
+#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
+#define GZEOF(fh) feof(fh)
+/* this is just the redefinition of a libz constant */
+#define Z_DEFAULT_COMPRESSION (-1)
+#endif
+
/* Initial buffer sizes used in zlib compression. */
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +55,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compress_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +67,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compress_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index fcc5f6bd05..cf9fad4706 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compress_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 233198afc0..fa84c31ecb 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +93,10 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +235,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +251,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compress_spec;
+
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compress_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +354,7 @@ RestoreArchive(Archive *AHX)
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -384,7 +383,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compress_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +739,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +969,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compress_spec.level = INT_MIN;
return opts;
}
@@ -1115,23 +1117,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compress_spec;
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compress_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compress_spec.algorithm));
switch (AH->format)
{
@@ -1485,60 +1492,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compress_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compress_spec);
if (!AH->OF)
{
@@ -1549,33 +1531,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1699,22 +1672,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2198,10 +2166,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2249,14 +2219,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compress_spec = compress_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2264,7 +2234,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3705,7 +3675,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compress_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3776,21 +3746,25 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compress_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compress_spec.level = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compress_spec.level = Z_DEFAULT_COMPRESSION;
+ if (AH->compress_spec.level != INT_MIN)
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+#else
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 084cd87e8d..341d406515 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compress_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 1023fea01b..43ccbe1339 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 3f46f7988a..17c7130c75 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compress_spec;
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compress_spec;
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index bfc49b66d2..085f5c7f20 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +801,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +888,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index c871cb727d..dd214e7670 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -54,7 +54,9 @@
#include "catalog/pg_subscription.h"
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
+#include "common/compression.h"
#include "common/connect.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -163,6 +165,8 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression_option(const char *opt,
+ pg_compress_specification *compress_spec);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -339,8 +343,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ pg_compress_specification compress_spec = {0};
+ bool user_compression_defined = false;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -560,10 +565,10 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression_option(optarg, &compress_spec))
exit_nicely(1);
+ user_compression_defined = true;
break;
case 0:
@@ -686,23 +691,23 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /*
+ * Custom and directory formats are compressed by default (zlib), others
+ * not
+ */
+ if (user_compression_defined == false)
{
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ compress_spec.level = INT_MIN;
#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ compress_spec.level = Z_DEFAULT_COMPRESSION;
+ }
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -715,8 +720,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compress_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -947,10 +952,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compress_spec = compress_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -997,7 +999,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1257,6 +1260,110 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+/*
+ * Interprets a compression option of the format 'method[:LEVEL]' of legacy just
+ * '[LEVEL]'. In the later format, gzip is implied. The parsed method and level
+ * are returned in pg_compress_specification. In case of error, the function
+ * returns false.
+ */
+static bool
+parse_compression_option(const char *opt,
+ pg_compress_specification *compress_spec)
+{
+ char *method;
+ const char *sep;
+ int methodlen;
+ bool supports_compression = true;
+ bool res = true;
+
+ /* set defaults */
+ compress_spec->algorithm = PG_COMPRESSION_NONE;
+ compress_spec->level = INT_MIN;
+
+ /* find the separator if exists */
+ sep = strchr(opt, ':');
+
+ /*
+ * If there is no separator, then it is either a legacy format, or only
+ * the method has been passed.
+ */
+ if (!sep)
+ {
+ if (strspn(opt, "-0123456789") == strlen(opt))
+ {
+ res = option_parse_int(opt, "-Z/--compress", 0, 9,
+ &(compress_spec->level));
+ compress_spec->algorithm = (compress_spec->level > 0) ?
+ PG_COMPRESSION_GZIP :
+ PG_COMPRESSION_NONE;
+ }
+ else
+ {
+ method = pg_strdup(opt);
+ res = parse_compress_algorithm(method,
+ &compress_spec->algorithm);
+ if (!res)
+ pg_log_error("invalid compression method \"%s\" (gzip, none)",
+ method);
+
+ pg_free(method);
+ }
+ }
+ else
+ {
+ /* otherwise, it should be method:LEVEL */
+ methodlen = sep - opt + 1;
+ method = pg_malloc0(methodlen);
+ snprintf(method, methodlen, "%.*s", methodlen - 1, opt);
+
+ res = parse_compress_algorithm(method, &compress_spec->algorithm);
+ if (res)
+ {
+ char *error_detail = NULL;
+ char *detail;
+
+ sep++;
+ detail = pg_strdup(sep);
+ parse_compress_specification(compress_spec->algorithm,
+ detail,
+ compress_spec);
+ error_detail = validate_compress_specification(compress_spec);
+ if (error_detail != NULL)
+ {
+ pg_log_error("invalid compression specification: %s", error_detail);
+ res = false;
+ }
+
+ pg_free(detail);
+ }
+
+ pg_free(method);
+ }
+
+ /* there is no need to check further when an error is already detected */
+ if (!res)
+ return false;
+
+ /* verify that the requested compression is supported */
+
+ if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
+ compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ supports_compression = false;
+
+#ifndef HAVE_LIBZ
+ if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ compress_spec->algorithm = PG_COMPRESSION_NONE;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..e1f1a8801c 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -120,6 +120,16 @@ command_fails_like(
qr/\Qpg_restore: error: cannot specify both --single-transaction and multiple jobs\E/,
'pg_restore: cannot specify both --single-transaction and multiple jobs');
+command_fails_like(
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level');
+
command_fails_like(
[ 'pg_dump', '-Z', '-1' ],
qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 1f08716f69..a02b578750 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
--
2.34.1
[text/x-patch] v7-0003-Add-LZ4-compression-in-pg_-dump-restore.patch (31.8K, ../../u7dLpRH_6r_rGyqNWHSDx2Wz5g1virAU3-f4XMC2BwiR81Tegqm4Sxtte0YAe77je9KVMa4jVxDNSt3XkYLYpyOoabCid1Bm7BZxwoitcmY=@pm.me/4-v7-0003-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 8f07443e04d2e610414d023e8aa443d8aa785dfd Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 5 Jul 2022 12:34:08 +0000
Subject: [PATCH v7 3/3] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding to
fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression is
still stored in the dump, though admittedly is of no apparent use.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 41 +-
src/bin/pg_dump/compress_lz4.c | 593 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 9 +
src/bin/pg_dump/pg_backup_archiver.c | 73 ++--
src/bin/pg_dump/pg_backup_archiver.h | 4 +-
src/bin/pg_dump/pg_dump.c | 8 +-
src/bin/pg_dump/t/001_basic.pl | 2 +-
src/bin/pg_dump/t/002_pg_dump.pl | 69 +++-
10 files changed, 769 insertions(+), 55 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 94885d0812..1097488b83 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 2d777ec213..1f7fd8d1f0 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
@@ -24,6 +25,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 1948ee3d57..4ef1cb5291 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
/*----------------------
@@ -125,6 +128,9 @@ AllocateCompressor(const pg_compress_specification compress_spec,
case PG_COMPRESSION_GZIP:
InitCompressorGzip(cs, compress_spec.level);
break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressorLZ4(cs, compress_spec.level);
+ break;
default:
pg_fatal("invalid compression method");
break;
@@ -175,6 +181,7 @@ free_keep_errno(void *p)
/*
* Compression None implementation
*/
+
static size_t
_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
@@ -310,6 +317,9 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
case PG_COMPRESSION_GZIP:
InitCompressGzip(CFH, compress_spec.level);
break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressLZ4(CFH, compress_spec.level);
+ break;
default:
pg_fatal("invalid compression method");
break;
@@ -322,12 +332,12 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -363,6 +373,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..6f4680c344
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,593 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..fbec9a508d
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, int compressionLevel);
+extern void InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 186510c235..225c226a6e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -353,6 +353,7 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
CompressFileHandle *sav;
@@ -382,17 +383,28 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if ((AH->compress_spec.algorithm == PG_COMPRESSION_GZIP ||
+ AH->compress_spec.algorithm == PG_COMPRESSION_LZ4) &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compress_algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -2028,6 +2040,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2054,30 +2078,20 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
+ return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3684,6 +3698,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
WriteInt(AH, AH->compress_spec.level);
+ AH->WriteBytePtr(AH, AH->compress_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3765,14 +3780,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compress_spec.level = Z_DEFAULT_COMPRESSION;
- if (AH->compress_spec.level != INT_MIN)
+ if (AH->version >= K_VERS_1_15)
+ AH->compress_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->compress_spec.level != INT_MIN)
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
+
#ifndef HAVE_LIBZ
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ {
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
-#else
- AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->compress_spec.level = 0;
+ }
#endif
-
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 341d406515..f577e8fc61 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,12 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compressionMethod
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index dd214e7670..e1253f7291 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1303,7 +1303,7 @@ parse_compression_option(const char *opt,
res = parse_compress_algorithm(method,
&compress_spec->algorithm);
if (!res)
- pg_log_error("invalid compression method \"%s\" (gzip, none)",
+ pg_log_error("invalid compression method \"%s\" (gzip, lz4, none)",
method);
pg_free(method);
@@ -1345,8 +1345,8 @@ parse_compression_option(const char *opt,
return false;
/* verify that the requested compression is supported */
-
if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
+ compress_spec->algorithm != PG_COMPRESSION_LZ4 &&
compress_spec->algorithm != PG_COMPRESSION_GZIP)
supports_compression = false;
@@ -1354,6 +1354,10 @@ parse_compression_option(const char *opt,
if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
#endif
+#ifndef USE_LZ4
+ if (compression_spec->algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
if (!supports_compression)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index e1f1a8801c..3803370350 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,7 +122,7 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index a02b578750..9109d72f8a 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -116,6 +116,67 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=1', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4057,11 +4118,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-07-05 15:13 ` Justin Pryzby <[email protected]>
2 siblings, 0 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-07-05 15:13 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; Rachel Heaton <[email protected]>; Michael Paquier <[email protected]>; Robert Haas <[email protected]>
This is a review of 0001.
On Tue, Jul 05, 2022 at 01:22:47PM +0000, [email protected] wrote:
> Simply this patchset had started to divert
> heavily already based on comments from Mr. Paquier who had already requested for
> the APIs to be refactored to use function pointers. This is happening in 0002 of
> the patchset.
I said something about reducing ifdefs, but I'm having trouble finding what
Michael said about this ?
> > On Sat, Mar 26, 2022 at 11:21:56AM -0500, Justin Pryzby wrote:
> >
> > > LZ4F_HEADER_SIZE_MAX isn't defined in old LZ4.
> > >
> > > I ran into that on an ubuntu LTS, so I don't think it's so old that it
> > > shouldn't be handled more gracefully. LZ4 should either have an explicit
> > > version check, or else shouldn't depend on that feature (or should define a
> > > safe fallback version if the library header doesn't define it).
> > > https://packages.ubuntu.com/liblz4-1
The constant still seems to be used without defining a fallback or a minimum version.
> > > 0003: typo: of legacy => or legacy
This is still there
> > > You renamed this:
> > >
> > > |- COMPR_ALG_LIBZ
> > > |-} CompressionAlgorithm;
> > > |+ COMPRESSION_GZIP,
> > > |+} CompressionMethod;
> > >
> > > ..But I don't think that's an improvement. If you were to change it, it should
> > > say something like PGDUMP_COMPRESS_ZLIB, since there are other compression
> > > structs and typedefs. zlib is not idential to gzip, which uses a different
> > > header, so in WriteDataToArchive(), LIBZ is correct, and GZIP is incorrect.
This comment still applies - zlib's gz* functions are "gzip" but the others are
"zlib". https://zlib.net/manual.html
That affects both the 0001 and 0002 patches.
Actually, I think that "gzip" should not be the name of the user-facing option,
since (except for "plain" format) it isn't using gzip.
+Robert, since this suggests amending parse_compress_algorithm(). Maybe "zlib"
should be parsed the same way as "gzip" - I don't think we ever expose both to
a user, but in some cases (basebackup and pg_dump -Fp -Z1) the output is "gzip"
and in some cases NO it's zlib (pg_dump -Fc -Z1).
> > > The cf* changes in pg_backup_archiver could be split out into a separate
> > > commit. It's strictly a code simplification - not just preparation for more
> > > compression algorithms. The commit message should "See also:
> > > bf9aa490db24b2334b3595ee33653bf2fe39208c".
I still think this could be an early, 0000 patch.
> > > freebsd/cfbot is failing.
This is still failing for bsd, windows and compiler warnings.
Windows also has compiler warnings.
http://cfbot.cputube.org/georgios-kokolatos.html
Please see: src/tools/ci/README, which you can use to run check-world on 4 OS
by pushing a branch to github.
> > > I suggested off-list to add an 0099 patch to change LZ4 to the default, to
> > > exercise it more on CI.
What about this ? I think the patch needs to pass CI on all 4 OS with
default=zlib and default=lz4.
> > On Sat, Mar 26, 2022 at 01:33:36PM -0500, Justin Pryzby wrote:
> @@ -254,7 +251,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
> Archive *
> OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
> {
> - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
> + ArchiveHandle *AH;
> + pg_compress_specification compress_spec;
Should this be initialized to {0} ?
> @@ -969,6 +969,8 @@ NewRestoreOptions(void)
> opts->format = archUnknown;
> opts->cparams.promptPassword = TRI_DEFAULT;
> opts->dumpSections = DUMP_UNSECTIONED;
> + opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
> + opts->compress_spec.level = INT_MIN;
Why INT_MIN ?
> @@ -1115,23 +1117,28 @@ PrintTOCSummary(Archive *AHX)
> ArchiveHandle *AH = (ArchiveHandle *) AHX;
> RestoreOptions *ropt = AH->public.ropt;
> TocEntry *te;
> + pg_compress_specification out_compress_spec;
Should have {0} ?
I suggest to write it like my 2020 patch for this, which says:
no_compression = {0};
> /* Open stdout with no compression for AH output handle */
> - AH->gzOut = 0;
> - AH->OF = stdout;
> + out_compress_spec.algorithm = PG_COMPRESSION_NONE;
> + AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
Ideally this should check the success of dup().
> @@ -3776,21 +3746,25 @@ ReadHead(ArchiveHandle *AH)
> + if (AH->compress_spec.level != INT_MIN)
Why is it testing the level and not the algorithm ?
> --- a/src/bin/pg_dump/pg_backup_custom.c
> +++ b/src/bin/pg_dump/pg_backup_custom.c
> @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
> _WriteByte(AH, BLK_DATA); /* Block type */
> WriteInt(AH, te->dumpId); /* For sanity check */
>
> - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
> + ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
Is it necessary to rename the data structure ?
If not, this file can remain unchanged.
> --- a/src/bin/pg_dump/pg_backup_directory.c
> +++ b/src/bin/pg_dump/pg_backup_directory.c
> @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
> if (AH->mode == archModeWrite)
> {
> cfp *tocFH;
> + pg_compress_specification compress_spec;
Should use {0} ?
> @@ -639,12 +642,14 @@ static void
> _StartBlobs(ArchiveHandle *AH, TocEntry *te)
> {
> lclContext *ctx = (lclContext *) AH->formatData;
> + pg_compress_specification compress_spec;
Same
> + /*
> + * Custom and directory formats are compressed by default (zlib), others
> + * not
> + */
> + if (user_compression_defined == false)
Should be: !user_compression_defined
Your 0001+0002 patches (without 0003) fail to compile:
pg_backup_directory.c: In function ‘_ReadByte’:
pg_backup_directory.c:519:12: error: ‘CompressFileHandle’ {aka ‘struct CompressFileHandle’} has no member named ‘_IO_getc’
519 | return CFH->getc(CFH);
| ^~
pg_backup_directory.c:520:1: warning: control reaches end of non-void function [-Wreturn-type]
520 | }
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-07-21 05:04 ` Michael Paquier <[email protected]>
2022-08-02 18:42 ` Re: Add LZ4 compression in pg_dump Jacob Champion <[email protected]>
2 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-07-21 05:04 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Tue, Jul 05, 2022 at 01:22:47PM +0000, [email protected] wrote:
> I have updated for "some" of the comments. This is not an unwillingness to
> incorporate those specific comments. Simply this patchset had started to divert
> heavily already based on comments from Mr. Paquier who had already requested for
> the APIs to be refactored to use function pointers. This is happening in 0002 of
> the patchset. 0001 of the patchset is using the new compression.h under common.
>
> This patchset should be considered a late draft, as commentary, documentation,
> and some finer details are not yet finalized; because I am expecting the proposed
> refactor to receive a wealth of comments. It would be helpful to understand if
> the proposed direction is something worth to be worked upon, before moving to the
> finer details.
I have read through the patch set, and I like a lot the separation you
are doing here with CompressFileHandle where a compression method has
to specify a full set of callbacks depending on the actions that need
to be taken. One advantage, as you patch shows, is that you reduce
the dependency of each code path depending on the compression method,
with #ifdefs and such located mostly into their own file structure, so
as adding a new compression method becomes really easier. These
callbacks are going to require much more documentation to describe
what anybody using them should expect from them, and perhaps they
could be renamed in a more generic way as the currect names come from
POSIX (say read_char(), read_string()?), even if this patch has just
inherited the names coming from pg_dump itself, but this can be tuned
over and over.
The split into three parts as of 0001 to plug into pg_dump the new
compression option set, 0002 to introduce the callbacks and 0003 to
add LZ4, building on the two first parts, makes sense to me. 0001 and
0002 could be done in a reversed order as they are mostly independent,
this order is fine as-is.
In short, I am fine with the proposed approach.
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compressionMethod
+ * in header */
Indeed, the dump format needs a version bump for this information.
+static bool
+parse_compression_option(const char *opt,
+ pg_compress_specification *compress_spec)
This parsing logic in pg_dump.c looks a lot like what pg_receivewal.c
does with its parse_compress_options() where, for compatibility:
- If only a number is given:
-- Assume no compression if level is 0.
-- Assume gzip with given compression if level > 0.
- If a string is found, assume a full spec, with optionally a level.
So some consolidation could be done between both.
By the way, I can see that GZCLOSE(), etc. are still defined in
compress_io.h but they are not used.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-07-21 05:04 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-08-02 18:42 ` Jacob Champion <[email protected]>
2022-08-05 14:23 ` Re: Add LZ4 compression in pg_dump Georgios Kokolatos <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Jacob Champion @ 2022-08-02 18:42 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
This entry has been waiting on author input for a while (our current
threshold is roughly two weeks), so I've marked it Returned with
Feedback.
Once you think the patchset is ready for review again, you (or any
interested party) can resurrect the patch entry by visiting
https://commitfest.postgresql.org/38/3571/
and changing the status to "Needs Review", and then changing the
status again to "Move to next CF". (Don't forget the second step;
hopefully we will have streamlined this in the near future!)
Thanks,
--Jacob
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-07-21 05:04 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-08-02 18:42 ` Re: Add LZ4 compression in pg_dump Jacob Champion <[email protected]>
@ 2022-08-05 14:23 ` Georgios Kokolatos <[email protected]>
0 siblings, 0 replies; 67+ messages in thread
From: Georgios Kokolatos @ 2022-08-05 14:23 UTC (permalink / raw)
To: [email protected]; +Cc: Georgios Kokolatos <[email protected]>; Rachel Heaton <[email protected]>
Thank you for your work during commitfest.
The patch is still in development. Given vacation status, expect the next patches to be ready for the November commitfest.
For now it has moved to the September one. Further action will be taken then as needed.
Enjoy the rest of the summer!
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-11-02 13:28 ` Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-11-02 13:28 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; Rachel Heaton <[email protected]>; Michael Paquier <[email protected]>
Checking if you'll be able to submit new patches soon ?
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-11-06 14:53 ` [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-11-06 14:53 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>; Michael Paquier <[email protected]>
On Wed, Nov 2, 2022 at 14:28, Justin Pryzby <[email protected]> wrote:
> Checking if you'll be able to submit new patches soon ?
Thank you for checking up. Expect new versions within this commitfest cycle.
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-11-20 17:26 ` Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-11-20 17:26 UTC (permalink / raw)
To: [email protected]; Georgios Kokolatos <[email protected]>; +Cc: [email protected]; Rachel Heaton <[email protected]>; Michael Paquier <[email protected]>
On Fri, Aug 05, 2022 at 02:23:45PM +0000, Georgios Kokolatos wrote:
> Thank you for your work during commitfest.
>
> The patch is still in development. Given vacation status, expect the next patches to be ready for the November commitfest.
> For now it has moved to the September one. Further action will be taken then as needed.
On Sun, Nov 06, 2022 at 02:53:12PM +0000, [email protected] wrote:
> On Wed, Nov 2, 2022 at 14:28, Justin Pryzby <[email protected]> wrote:
> > Checking if you'll be able to submit new patches soon ?
>
> Thank you for checking up. Expect new versions within this commitfest cycle.
Hi,
I think this patch record should be closed for now. You can re-open the
existing patch record once a patch is ready to be reviewed.
The commitfest is a time for committing/reviewing patches that were
previously submitted, but there's no new patch since July. Making a
patch available for review at the start of the commitfest seems like a
requirement for current patch records (same as for new patch records).
I wrote essentially the same patch as your early patches 2 years ago
(before postgres was ready to consider new compression algorithms), so
I'm happy to review a new patch when it's available, regardless of its
status in the cfapp.
BTW, some of my own review comments from March weren't addressed.
Please check. Also, in February, I asked if you knew how to use
cirrusci to run checks on cirrusci, but the patches still had
compilation errors and warnings on various OS.
https://cirrus-ci.com/github/postgresql-cfbot/postgresql/commitfest/39/3571
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-11-20 23:13 ` Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-11-20 23:13 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: [email protected]; Georgios Kokolatos <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Sun, Nov 20, 2022 at 11:26:11AM -0600, Justin Pryzby wrote:
> I think this patch record should be closed for now. You can re-open the
> existing patch record once a patch is ready to be reviewed.
Indeed. As of things are, this is just a dead entry in the CF which
would be confusing. I have marked it as RwF.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-11-22 10:00 ` [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-23 20:52 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 2 replies; 67+ messages in thread
From: [email protected] @ 2022-11-22 10:00 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Monday, November 21st, 2022 at 12:13 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Sun, Nov 20, 2022 at 11:26:11AM -0600, Justin Pryzby wrote:
>
> > I think this patch record should be closed for now. You can re-open the
> > existing patch record once a patch is ready to be reviewed.
>
>
> Indeed. As of things are, this is just a dead entry in the CF which
> would be confusing. I have marked it as RwF.
Thank you for closing it.
For the record I am currently working on it simply unsure if I should submit
WIP patches and add noise to the list or wait until it is in a state that I
feel that the comments have been addressed.
A new version that I feel that is in a decent enough state for review should
be ready within this week. I am happy to drop the patch if you think I should
not work on it though.
Cheers,
//Georgios
> --
> Michael
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-11-22 10:49 ` Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
1 sibling, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-11-22 10:49 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Tue, Nov 22, 2022 at 10:00:47AM +0000, [email protected] wrote:
> A new version that I feel that is in a decent enough state for review should
> be ready within this week. I am happy to drop the patch if you think I should
> not work on it though.
If you can post a new version of the patch, that's fine, of course.
I'll be happy to look over it more.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-11-28 16:32 ` [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-11-28 16:32 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Tuesday, November 22nd, 2022 at 11:49 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Tue, Nov 22, 2022 at 10:00:47AM +0000, [email protected] wrote:
>
> > A new version that I feel that is in a decent enough state for review should
> > be ready within this week. I am happy to drop the patch if you think I should
> > not work on it though.
>
>
> If you can post a new version of the patch, that's fine, of course.
> I'll be happy to look over it more.
Thank you Michael (and Justin). Allow me to present v8.
The focus of this version of this series is 0001 and 0002.
Admittedly 0001 could be presented in a separate thread though given its size and
proximity to the topic, I present it here.
In an earlier review you spotted the similarity between pg_dump's and pg_receivewal's
parsing of compression options. However there exists a substantial difference in the
behaviour of the two programs; one treats the lack of support for the requested
algorithm as a fatal error, whereas the other does not. The existing functions in
common/compression.c do not account for the later. 0002 proposes an implementation
for this. It's usefulness is shown in 0003.
Please consider 0003-0005 as work in progress. They are differences from v7 yet they
may contain unaddressed comments for now.
A welcome feedback would be in splitting and/or reordering of 0003-0005. I think
that they now split in coherent units and are presented in a logical order. Let me
know if you disagree and where should the breakpoints be.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v8-0003-Prepare-pg_dump-for-additional-compression-method.patch (50.5K, ../../O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me/2-v8-0003-Prepare-pg_dump-for-additional-compression-method.patch)
download | inline diff:
From 337f19a52f164a22fbf974b8a749f3b895a339b4 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 28 Nov 2022 15:33:49 +0000
Subject: [PATCH v8 3/5] Prepare pg_dump for additional compression methods
This commmit does some of the heavy lifting required for additional compression
methods.
First it is teaching pg_dump.c about the definitions and interfaces found in
common/compression.h. Then it is propagating those throughout the code.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 431 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 20 +-
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 192 ++++++------
src/bin/pg_dump/pg_backup_archiver.h | 37 +--
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 98 ++++--
src/bin/pg_dump/t/001_basic.pl | 26 +-
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
12 files changed, 512 insertions(+), 362 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8b9d9f4cad..3fb8fdce81 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..4a8fc1e306 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -64,7 +68,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_algorithm compress_algorithm;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +78,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +94,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compress_algorithm = compress_spec.algorithm;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compress_spec.level);
#endif
return cs;
@@ -154,21 +128,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compress_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +156,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compress_algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +180,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compress_algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +410,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_algorithm compress_algorithm;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -452,21 +442,25 @@ cfp *
cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compress_spec = {0};
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compress_spec);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compress_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +473,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compress_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compress_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -509,60 +504,96 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_algorithm compress_algorithm, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compress_algorithm = compress_algorithm;
+
+ switch (compress_algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(path, -1, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(NULL, fd, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +603,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +665,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +698,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compress_algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..d6335fff02 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compress_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compress_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index e8b7898297..61c412c8cb 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compress_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index f39c0fa36f..304cc072ca 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +93,10 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +235,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +251,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compress_spec = {0};
+
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compress_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -354,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -383,16 +383,23 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ supports_compression = true;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -459,8 +466,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compress_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +746,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +976,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compress_spec.level = INT_MIN;
return opts;
}
@@ -1115,23 +1124,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compress_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compress_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compress_spec.algorithm));
switch (AH->format)
{
@@ -1485,60 +1499,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compress_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compress_spec);
if (!AH->OF)
{
@@ -1549,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1699,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2198,10 +2173,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2249,14 +2226,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compress_spec = compress_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2264,7 +2241,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3669,7 +3646,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compress_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3740,21 +3717,26 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compress_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compress_spec.level = ReadInt(AH);
+
+ if (AH->compress_spec.level != 0)
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 42687c4ec8..d2930949ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compress_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index a0a55a1edd..6a2112c45f 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 798182b6f7..7d2cddbb2c 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 402b93c610..d773c291c8 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +801,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +888,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index da427f4d4a..a97a0f3a84 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -54,8 +54,10 @@
#include "catalog/pg_subscription.h"
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
+#include "common/compression.h"
#include "common/connect.h"
#include "common/relpath.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -164,6 +166,8 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression(const char *opt,
+ pg_compress_specification *compress_spec);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -340,8 +344,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ pg_compress_specification compress_spec = {0};
+ bool user_compression_defined = false;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -561,10 +566,10 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression(optarg, &compress_spec))
exit_nicely(1);
+ user_compression_defined = true;
break;
case 0:
@@ -687,23 +692,20 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /*
+ * Custom and directory formats are compressed by default (zlib), others
+ * not
+ */
+ if (user_compression_defined == false)
{
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, &compress_spec);
#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ parse_compress_specification(PG_COMPRESSION_GZIP, NULL,
+ &compress_spec);
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -716,8 +718,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compress_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -948,10 +950,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compress_spec = compress_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -998,7 +997,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1258,6 +1258,60 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+/*
+ * Interprets and validates a compression option using the common compression
+ * parsing functions. If the requested compression is not available then the
+ * archives are uncompressed.
+ */
+static bool
+parse_compression(const char *opt, pg_compress_specification *compress_spec)
+{
+ char *algorithm_str = NULL;
+ char *level_str = NULL;
+ char *validation_error = NULL;
+ bool supports_compression = true;
+
+ compress_spec->algorithm = PG_COMPRESSION_NONE;
+ compress_spec->level = 0;
+
+ parse_compress_user_options(opt, &algorithm_str, &level_str);
+ if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
+ {
+ pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ algorithm_str);
+ return false;
+ }
+
+ parse_compress_specification(compress_spec->algorithm, level_str,
+ compress_spec);
+ validation_error = validate_compress_specification(compress_spec);
+ if (validation_error)
+ {
+ pg_log_error("invalid compression specification: %s", validation_error);
+ return false;
+ }
+
+ /* Switch off unsupported compressions that made it through parsing */
+ if (test_compress_support(compress_spec))
+ supports_compression = false;
+
+ /* Also switch off unimplemented compressions */
+ if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
+ compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ supports_compression = false;
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, compress_spec);
+ }
+
+ pg_free(algorithm_str);
+ pg_free(level_str);
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..fffb9e075b 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,12 +121,27 @@ command_fails_like(
'pg_restore: cannot specify both --single-transaction and multiple jobs');
command_fails_like(
- [ 'pg_dump', '-Z', '-1' ],
- qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
- 'pg_dump: -Z/--compress must be in range');
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level');
+
+command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
if (check_pg_config("#define HAVE_LIBZ 1"))
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '15' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
@@ -134,6 +149,11 @@ if (check_pg_config("#define HAVE_LIBZ 1"))
}
else
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '-1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at 0)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
# --jobs > 1 forces an error with tar format.
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 8dc1f0eccb..e97d086956 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
--
2.34.1
[text/x-patch] v8-0001-Export-gzip-program-to-pg_dump-tap-tests.patch (658B, ../../O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me/3-v8-0001-Export-gzip-program-to-pg_dump-tap-tests.patch)
download | inline diff:
From f5125b4d4ca5bfc7ef7f68b7dbe4a1c7777de6eb Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 28 Nov 2022 15:09:22 +0000
Subject: [PATCH v8 1/5] Export gzip program to pg_dump tap tests
---
src/bin/pg_dump/meson.build | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index e66f632b54..6cff2a6c3d 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -88,6 +88,9 @@ tests += {
't/003_pg_dump_with_server.pl',
't/010_dump_connstr.pl',
],
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ },
},
}
--
2.34.1
[text/x-patch] v8-0002-Make-the-pg_receivewal-compression-parsing-functi.patch (10.0K, ../../O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me/4-v8-0002-Make-the-pg_receivewal-compression-parsing-functi.patch)
download | inline diff:
From a4ce723e6719ed80c36fd0c4fd85c962b6b25a45 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 28 Nov 2022 15:18:27 +0000
Subject: [PATCH v8 2/5] Make the pg_receivewal compression parsing function
common
Also and relax parsing errors in the helper functions and re-introduce those as
an independed function.
As it is shown in the rest of the patch series, there is a lot of duplication
between pg_dump's parsing of compression options and pg_receivewal's. Now the
core work is done in common. However pg_dump would not error out if the
requested compression algorithm is not supported by the build, whereas other
callers will error out. Also it seems a bit weird for only one of the parsing
functions for compressions to error out on missing support and that one to not
be the one responsible for identifying the compression algorithm.
A new function is added to test the support of the algorithm allowing the user
to tune the behaviour.
---
src/backend/backup/basebackup.c | 1 +
src/bin/pg_basebackup/pg_basebackup.c | 1 +
src/bin/pg_basebackup/pg_receivewal.c | 65 +-------------
src/common/compression.c | 119 ++++++++++++++++++++++----
src/include/common/compression.h | 3 +
5 files changed, 111 insertions(+), 78 deletions(-)
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
index 74fb529380..70cd720823 100644
--- a/src/backend/backup/basebackup.c
+++ b/src/backend/backup/basebackup.c
@@ -942,6 +942,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
parse_compress_specification(opt->compression, compression_detail_str,
&opt->compression_specification);
+ (void) test_compress_support(&opt->compression_specification);
error_detail =
validate_compress_specification(&opt->compression_specification);
if (error_detail != NULL)
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 22836ca01a..cdd32c9763 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -2556,6 +2556,7 @@ main(int argc, char **argv)
compression_algorithm);
parse_compress_specification(alg, compression_detail, &client_compress);
+ (void) test_compress_support(&client_compress);
error_detail = validate_compress_specification(&client_compress);
if (error_detail != NULL)
pg_fatal("invalid compression specification: %s",
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index 63207ca025..d4a3a4213d 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -57,8 +57,6 @@ static XLogRecPtr endpos = InvalidXLogRecPtr;
static void usage(void);
-static void parse_compress_options(char *option, char **algorithm,
- char **detail);
static DIR *get_destination_dir(char *dest_folder);
static void close_destination_dir(DIR *dest_dir, char *dest_folder);
static XLogRecPtr FindStreamingStart(uint32 *tli);
@@ -109,65 +107,6 @@ usage(void)
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
-/*
- * Basic parsing of a value specified for -Z/--compress
- *
- * The parsing consists of a METHOD:DETAIL string fed later on to a more
- * advanced routine in charge of proper validation checks. This only extracts
- * METHOD and DETAIL. If only an integer is found, the method is implied by
- * the value specified.
- */
-static void
-parse_compress_options(char *option, char **algorithm, char **detail)
-{
- char *sep;
- char *endp;
- long result;
-
- /*
- * Check whether the compression specification consists of a bare integer.
- *
- * For backward-compatibility, assume "none" if the integer found is zero
- * and "gzip" otherwise.
- */
- result = strtol(option, &endp, 10);
- if (*endp == '\0')
- {
- if (result == 0)
- {
- *algorithm = pstrdup("none");
- *detail = NULL;
- }
- else
- {
- *algorithm = pstrdup("gzip");
- *detail = pstrdup(option);
- }
- return;
- }
-
- /*
- * Check whether there is a compression detail following the algorithm
- * name.
- */
- sep = strchr(option, ':');
- if (sep == NULL)
- {
- *algorithm = pstrdup(option);
- *detail = NULL;
- }
- else
- {
- char *alg;
-
- alg = palloc((sep - option) + 1);
- memcpy(alg, option, sep - option);
- alg[sep - option] = '\0';
-
- *algorithm = alg;
- *detail = pstrdup(sep + 1);
- }
-}
/*
* Check if the filename looks like a WAL file, letting caller know if this
@@ -786,8 +725,8 @@ main(int argc, char **argv)
verbose++;
break;
case 'Z':
- parse_compress_options(optarg, &compression_algorithm_str,
- &compression_detail);
+ parse_compress_user_options(optarg, &compression_algorithm_str,
+ &compression_detail);
break;
/* action */
case 1:
diff --git a/src/common/compression.c b/src/common/compression.c
index df5b627834..57c23221a2 100644
--- a/src/common/compression.c
+++ b/src/common/compression.c
@@ -39,6 +39,66 @@
static int expect_integer_value(char *keyword, char *value,
pg_compress_specification *result);
+/*
+ * Basic parsing of a value specified for -Z/--compress
+ *
+ * The parsing consists of a METHOD:DETAIL string fed later on to a more
+ * advanced routine in charge of proper validation checks. This only extracts
+ * METHOD and DETAIL. If only an integer is found, the method is implied by
+ * the value specified.
+ */
+void
+parse_compress_user_options(const char *option, char **algorithm, char **detail)
+{
+ char *sep;
+ char *endp;
+ long result;
+
+ /*
+ * Check whether the compression specification consists of a bare integer.
+ *
+ * For backward-compatibility, assume "none" if the integer found is zero
+ * and "gzip" otherwise.
+ */
+ result = strtol(option, &endp, 10);
+ if (*endp == '\0')
+ {
+ if (result == 0)
+ {
+ *algorithm = pstrdup("none");
+ *detail = NULL;
+ }
+ else
+ {
+ *algorithm = pstrdup("gzip");
+ *detail = pstrdup(option);
+ }
+ return;
+ }
+
+ /*
+ * Check whether there is a compression detail following the algorithm
+ * name.
+ */
+ sep = strchr(option, ':');
+ if (sep == NULL)
+ {
+ *algorithm = pstrdup(option);
+ *detail = NULL;
+ }
+ else
+ {
+ char *alg;
+
+ alg = palloc((sep - option) + 1);
+ memcpy(alg, option, sep - option);
+ alg[sep - option] = '\0';
+
+ *algorithm = alg;
+ *detail = pstrdup(sep + 1);
+ }
+}
+
/*
* Look up a compression algorithm by name. Returns true and sets *algorithm
* if the name is recognized. Otherwise returns false.
@@ -100,6 +160,9 @@ get_compress_algorithm_name(pg_compress_algorithm algorithm)
*
* Use validate_compress_specification() to find out whether a compression
* specification is semantically sensible.
+ *
+ * Does not test whether this build of PostgreSQL supports the requested
+ * compression method.
*/
void
parse_compress_specification(pg_compress_algorithm algorithm, char *specification,
@@ -123,30 +186,16 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
result->level = 0;
break;
case PG_COMPRESSION_LZ4:
-#ifdef USE_LZ4
result->level = 0; /* fast compression mode */
-#else
- result->parse_error =
- psprintf(_("this build does not support compression with %s"),
- "LZ4");
-#endif
break;
case PG_COMPRESSION_ZSTD:
#ifdef USE_ZSTD
result->level = ZSTD_CLEVEL_DEFAULT;
-#else
- result->parse_error =
- psprintf(_("this build does not support compression with %s"),
- "ZSTD");
#endif
break;
case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
result->level = Z_DEFAULT_COMPRESSION;
-#else
- result->parse_error =
- psprintf(_("this build does not support compression with %s"),
- "gzip");
#endif
break;
}
@@ -265,7 +314,8 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio
* and return -1.
*/
static int
-expect_integer_value(char *keyword, char *value, pg_compress_specification *result)
+expect_integer_value(char *keyword, char *value,
+ pg_compress_specification *result)
{
int ivalue;
char *ivalue_endp;
@@ -356,3 +406,42 @@ validate_compress_specification(pg_compress_specification *spec)
return NULL;
}
+
+/*
+ * Returns NULL if the compression algorithm is supported by this build.
+ * Otherwise, returns an error message. In the later case, the error is attached
+ * to pg_compress_specification unless another error preceeds it.
+ */
+char *
+test_compress_support(pg_compress_specification *spec)
+{
+ char *err = NULL;
+ switch (spec->algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifndef HAVE_LIBZ
+ err = psprintf(_("this build does not support compression with %s"),
+ "gzip");
+#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+#ifndef USE_LZ4
+ err = psprintf(_("this build does not support compression with %s"),
+ "LZ4");
+#endif
+ break;
+ case PG_COMPRESSION_ZSTD:
+#ifndef USE_ZSTD
+ err = psprintf(_("this build does not support compression with %s"),
+ "ZSTD");
+#endif
+ break;
+ }
+
+ if (err && !spec->parse_error)
+ spec->parse_error = err;
+
+ return err;
+}
diff --git a/src/include/common/compression.h b/src/include/common/compression.h
index 5d680058ed..eb30a7b547 100644
--- a/src/include/common/compression.h
+++ b/src/include/common/compression.h
@@ -33,6 +33,8 @@ typedef struct pg_compress_specification
char *parse_error; /* NULL if parsing was OK, else message */
} pg_compress_specification;
+extern void parse_compress_user_options(const char *option, char **algorithm,
+ char **detail);
extern bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm);
extern const char *get_compress_algorithm_name(pg_compress_algorithm algorithm);
@@ -40,6 +42,7 @@ extern void parse_compress_specification(pg_compress_algorithm algorithm,
char *specification,
pg_compress_specification *result);
+extern char *test_compress_support(pg_compress_specification *);
extern char *validate_compress_specification(pg_compress_specification *);
#endif
--
2.34.1
[text/x-patch] v8-0004-Introduce-Compressor-API-in-pg_dump.patch (53.9K, ../../O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me/5-v8-0004-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From b1fd402c6108ac88f705c7bebc75333983625221 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 28 Nov 2022 15:35:19 +0000
Subject: [PATCH v8 4/5] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 390 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 9 +
src/bin/pg_dump/compress_io.c | 817 ++++++--------------------
src/bin/pg_dump/compress_io.h | 69 ++-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 93 +--
src/bin/pg_dump/pg_backup_archiver.h | 4 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 85 +--
10 files changed, 765 insertions(+), 727 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..bc6d1abc77
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,390 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ int compressionLevel;
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, gzipcs->compressionLevel) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+typedef struct GzipData
+{
+ gzFile fp;
+ int compressionLevel;
+} GzipData;
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ size_t ret;
+
+ ret = gzread(gd->fp, ptr, size);
+ if (ret != size && !gzeof(gd->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gd->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzwrite(gd->fp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gd->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gd->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzgets(gd->fp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gd->fp);
+
+ save_errno = errno;
+ free(gd);
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzeof(gd->fp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gd->fp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ char mode_compression[32];
+
+ if (gd->compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, gd->compressionLevel);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gd->fp = gzdopen(dup(fd), mode_compression);
+ else
+ gd->fp = gzopen(path, mode_compression);
+
+ if (gd->fp == NULL)
+ return 1;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ GzipData *gd;
+
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ gd = pg_malloc0(sizeof(GzipData));
+ gd->compressionLevel = compressionLevel;
+
+ CFH->private = gd;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..ab0362c1f3
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, int compressionLevel);
+extern void InitCompressGzip(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 4a8fc1e306..3065bd76fa 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,113 +68,73 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_algorithm compress_algorithm;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+ free(buf);
+}
+
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compress_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compress_algorithm = compress_spec.algorithm;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, compress_spec.level);
-#endif
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
- ReadFunc readF)
-{
switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressorGzip(cs, compress_spec.level);
break;
default:
pg_fatal("invalid compression method");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compress_algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ return cs;
}
/*
@@ -180,243 +143,28 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compress_algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- free(cs);
- break;
-
- default:
- pg_fatal("invalid compression method");
- break;
- }
-}
-
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
- */
-
-static void
-InitCompressorZlib(CompressorState *cs, int level)
-{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
-}
-
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
-{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
-{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
-
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
-
- if (res == Z_STREAM_END)
- break;
- }
-}
-
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
-
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
-{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
-}
-#endif /* HAVE_LIBZ */
-
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
-{
- size_t cnt;
- char *buf;
- size_t buflen;
-
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
-
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
-
- free(buf);
-}
-
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->writeF(AH, data, dLen);
-}
-
-
/*----------------------
* Compressed stream API
*----------------------
*/
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- pg_compress_algorithm compress_algorithm;
- void *fp;
-};
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
+}
/* free() without changing errno; useful in several places below */
static void
@@ -429,392 +177,219 @@ free_keep_errno(void *p)
}
/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
+ * Compression None implementation
*/
-cfp *
-cfopen_read(const char *path, const char *mode)
+static size_t
+_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp;
- pg_compress_specification compress_spec = {0};
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, compress_spec);
- else
-#endif
- {
- compress_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compress_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
-
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
-}
+ if (size == 0)
+ return 0;
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
-{
- cfp *fp;
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
- if (compress_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compress_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ return ret;
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_algorithm compress_algorithm, int compressionLevel)
+static size_t
+_write(const void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
-
- fp->compress_algorithm = compress_algorithm;
-
- switch (compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compressionLevel != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compressionLevel);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
-
- return fp;
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
+static const char *
+_get_error(CompressFileHandle * CFH)
{
- return cfopen_internal(path, -1, mode,
- compress_spec.algorithm,
- compress_spec.level);
+ return strerror(errno);
}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compress_spec)
+static char *
+_gets(char *ptr, int size, CompressFileHandle * CFH)
{
- return cfopen_internal(NULL, fd, mode,
- compress_spec.algorithm,
- compress_spec.level);
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-int
-cfread(void *ptr, int size, cfp *fp)
+static int
+_getc(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret;
- if (size == 0)
- return 0;
-
- switch (fp->compress_algorithm)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
-
- default:
- pg_fatal("invalid compression method");
- break;
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
return ret;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+static int
+_close(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret = 0;
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ CFH->private = NULL;
+
+ if (fp)
+ ret = fclose(fp);
return ret;
}
-int
-cfgetc(cfp *fp)
+static int
+_eof(CompressFileHandle * CFH)
{
- int ret;
+ return feof((FILE *) CFH->private);
+}
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
+static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ Assert(CFH->private == NULL);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- return ret;
+ if (CFH->private == NULL)
+ return 1;
+
+ return 0;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+static int
+_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
{
- char *ret;
+ Assert(CFH->private == NULL);
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ return 0;
+}
- return ret;
+static void
+InitCompressNone(CompressFileHandle * CFH)
+{
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-int
-cfclose(cfp *fp)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compress_spec)
{
- int ret;
+ CompressFileHandle *CFH;
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
- switch (fp->compress_algorithm)
+ switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
+ InitCompressNone(CFH);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressGzip(CFH, compress_spec.level);
break;
default:
pg_fatal("invalid compression method");
break;
}
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
-int
-cfeof(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret;
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compress_spec = {0};
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- return ret;
-}
+ fname = strdup(path);
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
+ if (hasSuffix(fname, ".gz"))
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
+ bool exists;
+
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("not built with zlib support");
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
}
- return strerror(errno);
+ CFH = InitCompressFileHandle(compress_spec);
+ if (CFH->open(fname, -1, mode, CFH))
+ {
+ free_keep_errno(CFH);
+ CFH = NULL;
+ }
+ free_keep_errno(fname);
+
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle * CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d6335fff02..a986f5e6ee 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,61 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compress_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle * cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle * CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle * CFH);
+ int (*getc) (CompressFileHandle * CFH);
+ int (*eof) (CompressFileHandle * CFH);
+ int (*close) (CompressFileHandle * CFH);
+ const char *(*get_error) (CompressFileHandle * CFH);
+
+ void *private;
+};
+
-typedef struct cfp cfp;
+extern CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compress_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compress_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle * InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle * CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 6cff2a6c3d..041e08cd5e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 304cc072ca..09e20fb97b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle * SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1126,7 +1126,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compress_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1502,6 +1502,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1525,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compress_spec);
- else
- AH->OF = cfopen(filename, mode, compress_spec);
+ CFH = InitCompressFileHandle(compress_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1689,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2035,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2077,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2180,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2236,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,7 +3652,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compress_spec.level);
+ AH->WriteBytePtr(AH, AH->compress_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3718,7 +3724,9 @@ ReadHead(ArchiveHandle *AH)
AH->format, fmt);
AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compress_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
AH->compress_spec.level = AH->ReadBytePtr(AH);
@@ -3731,11 +3739,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
-
+ if (unsupported)
+ {
+ pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ }
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index d2930949ab..bb7fad2af1 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,12 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 6a2112c45f..49ec0e3816 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compress_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7d2cddbb2c..e1ce2f393b 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,9 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
+ CompressFileHandle *dataFH; /* currently open data file */
- cfp *blobsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *blobsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +198,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +218,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compress_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +346,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +371,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +386,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +435,7 @@ _LoadBlobs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +443,14 @@ _LoadBlobs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->blobsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the blobs TOC file line-by-line, and process each blob */
- while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char blobfname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +465,11 @@ _LoadBlobs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreBlob(AH, oid);
}
- if (!cfeof(ctx->blobsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +489,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +514,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +527,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +549,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +578,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +589,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compress_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +603,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +654,8 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
/* The blob TOC file is never compressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
- if (ctx->blobsTocFH == NULL)
+ ctx->blobsTocFH = InitCompressFileHandle(compress_spec);
+ if (ctx->blobsTocFH->open_write(fname, "ab", ctx->blobsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +672,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,17 +686,18 @@ static void
_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->blobsTocFH;
char buf[50];
int len;
/* Close the BLOB data file itself */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the blob in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->blobsTocFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
pg_fatal("could not write to blobs TOC file");
}
@@ -706,7 +711,7 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close blobs TOC file: %m");
ctx->blobsTocFH = NULL;
}
--
2.34.1
[text/x-patch] v8-0005-Add-LZ4-compression-in-pg_-dump-restore.patch (28.8K, ../../O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me/6-v8-0005-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From efd5acd0dc4dedab8fe823a9223404a43ba278d0 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 23 Nov 2022 16:59:02 +0000
Subject: [PATCH v8 5/5] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 41 +-
src/bin/pg_dump/compress_lz4.c | 593 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 9 +
src/bin/pg_dump/meson.build | 4 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 5 +-
src/bin/pg_dump/t/001_basic.pl | 2 +-
src/bin/pg_dump/t/002_pg_dump.pl | 69 +++-
10 files changed, 732 insertions(+), 30 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 3fb8fdce81..84d3778c99 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 3065bd76fa..29e2352c31 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -129,6 +132,9 @@ AllocateCompressor(const pg_compress_specification compress_spec,
case PG_COMPRESSION_GZIP:
InitCompressorGzip(cs, compress_spec.level);
break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressorLZ4(cs, compress_spec.level);
+ break;
default:
pg_fatal("invalid compression method");
break;
@@ -179,6 +185,7 @@ free_keep_errno(void *p)
/*
* Compression None implementation
*/
+
static size_t
_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
@@ -314,6 +321,9 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
case PG_COMPRESSION_GZIP:
InitCompressGzip(CFH, compress_spec.level);
break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressLZ4(CFH, compress_spec.level);
+ break;
default:
pg_fatal("invalid compression method");
break;
@@ -326,12 +336,12 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -367,6 +377,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..6f4680c344
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,593 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..fbec9a508d
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, int compressionLevel);
+extern void InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 041e08cd5e..f9065a3e84 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -91,6 +92,7 @@ tests += {
],
'env': {
'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
},
},
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 09e20fb97b..c9b053d572 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -394,6 +394,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2073,7 +2077,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2083,6 +2087,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3746,6 +3754,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
{
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index a97a0f3a84..93c69834f0 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1277,7 +1277,7 @@ parse_compression(const char *opt, pg_compress_specification *compress_spec)
parse_compress_user_options(opt, &algorithm_str, &level_str);
if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
{
- pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ pg_log_error("invalid compression method: \"%s\" (gzip, lz4, none)",
algorithm_str);
return false;
}
@@ -1297,7 +1297,8 @@ parse_compression(const char *opt, pg_compress_specification *compress_spec)
/* Also switch off unimplemented compressions */
if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
- compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ compress_spec->algorithm != PG_COMPRESSION_GZIP &&
+ compress_spec->algorithm != PG_COMPRESSION_LZ4)
supports_compression = false;
if (!supports_compression)
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index fffb9e075b..da266c3013 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,7 +122,7 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index e97d086956..88f0e83b43 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -116,6 +116,67 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=1', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4127,11 +4188,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-11-29 06:19 ` Michael Paquier <[email protected]>
2022-11-29 07:08 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 2 replies; 67+ messages in thread
From: Michael Paquier @ 2022-11-29 06:19 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Mon, Nov 28, 2022 at 04:32:43PM +0000, [email protected] wrote:
> The focus of this version of this series is 0001 and 0002.
>
> Admittedly 0001 could be presented in a separate thread though given its size and
> proximity to the topic, I present it here.
I don't mind. This was a hole in meson.build, so nice catch! I have
noticed a second defect with pg_verifybackup for all the commands, and
applied both at the same time.
> In an earlier review you spotted the similarity between pg_dump's and pg_receivewal's
> parsing of compression options. However there exists a substantial difference in the
> behaviour of the two programs; one treats the lack of support for the requested
> algorithm as a fatal error, whereas the other does not. The existing functions in
> common/compression.c do not account for the later. 0002 proposes an implementation
> for this. It's usefulness is shown in 0003.
In what does it matter? The logic in compression.c provides an error
when looking at a spec or validating it, but the caller is free to
consume it as it wants because this is shared between the frontend and
the backend, and that includes consuming it as a warning rather than a
ahrd failure. If we don't want to issue an error and force
non-compression if attempting to use a compression method not
supported in pg_dump, that's fine by me as a historical behavior, but
I don't see why these routines have any need to be split more as
proposed in 0002.
Saying that, I do agree that it would be nice to remove the
duplication between the option parsing of pg_basebackup and
pg_receivewal. Your patch is very close to that, actually, and it
occured to me that if we move the check on "server-" and "client-" in
pg_basebackup to be just before the integer-only check then we can
consolidate the whole thing.
Attached is an alternative that does not sacrifice the pluggability of
the existing routines while allowing 0003~ to still use them (I don't
really want to move around the checks on the supported build options
now in parse_compress_specification(), that was hard enough to settle
on this location). On top of that, pg_basebackup is able to cope with
the case of --compress=0 already, enforcing "none" (BaseBackup could
be simplified a bit more before StartLogStreamer). This refactoring
shaves a little bit of code.
> Please consider 0003-0005 as work in progress. They are differences from v7 yet they
> may contain unaddressed comments for now.
Okay.
--
Michael
Attachments:
[text/x-diff] v9-0001-Make-the-pg_receivewal-compression-parsing-functi.patch (7.7K, ../../[email protected]/2-v9-0001-Make-the-pg_receivewal-compression-parsing-functi.patch)
download | inline diff:
From 6fb2aa609348ad7df6f9c12da60c07aa96243965 Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Tue, 29 Nov 2022 15:17:27 +0900
Subject: [PATCH v9] Make the pg_receivewal compression parsing function common
Also and relax parsing errors in the helper functions and re-introduce those as
an independed function.
As it is shown in the rest of the patch series, there is a lot of duplication
between pg_dump's parsing of compression options and pg_receivewal's. Now the
core work is done in common. However pg_dump would not error out if the
requested compression algorithm is not supported by the build, whereas other
callers will error out. Also it seems a bit weird for only one of the parsing
functions for compressions to error out on missing support and that one to not
be the one responsible for identifying the compression algorithm.
A new function is added to test the support of the algorithm allowing the user
to tune the behaviour.
---
src/include/common/compression.h | 2 +
src/common/compression.c | 63 +++++++++++++++++++++++++++
src/bin/pg_basebackup/pg_basebackup.c | 49 ++++-----------------
src/bin/pg_basebackup/pg_receivewal.c | 61 --------------------------
4 files changed, 73 insertions(+), 102 deletions(-)
diff --git a/src/include/common/compression.h b/src/include/common/compression.h
index 5d680058ed..46855b1a3b 100644
--- a/src/include/common/compression.h
+++ b/src/include/common/compression.h
@@ -33,6 +33,8 @@ typedef struct pg_compress_specification
char *parse_error; /* NULL if parsing was OK, else message */
} pg_compress_specification;
+extern void parse_compress_options(const char *option, char **algorithm,
+ char **detail);
extern bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm);
extern const char *get_compress_algorithm_name(pg_compress_algorithm algorithm);
diff --git a/src/common/compression.c b/src/common/compression.c
index df5b627834..5274ba5ba8 100644
--- a/src/common/compression.c
+++ b/src/common/compression.c
@@ -356,3 +356,66 @@ validate_compress_specification(pg_compress_specification *spec)
return NULL;
}
+
+#ifdef FRONTEND
+
+/*
+ * Basic parsing of a value specified through a command-line option, commonly
+ * -Z/--compress.
+ *
+ * The parsing consists of a METHOD:DETAIL string fed later to
+ * parse_compress_specification(). This only extracts METHOD and DETAIL.
+ * If only an integer is found, the method is implied by the value specified.
+ */
+void
+parse_compress_options(const char *option, char **algorithm, char **detail)
+{
+ char *sep;
+ char *endp;
+ long result;
+
+ /*
+ * Check whether the compression specification consists of a bare integer.
+ *
+ * For backward-compatibility, assume "none" if the integer found is zero
+ * and "gzip" otherwise.
+ */
+ result = strtol(option, &endp, 10);
+ if (*endp == '\0')
+ {
+ if (result == 0)
+ {
+ *algorithm = pstrdup("none");
+ *detail = NULL;
+ }
+ else
+ {
+ *algorithm = pstrdup("gzip");
+ *detail = pstrdup(option);
+ }
+ return;
+ }
+
+ /*
+ * Check whether there is a compression detail following the algorithm
+ * name.
+ */
+ sep = strchr(option, ':');
+ if (sep == NULL)
+ {
+ *algorithm = pstrdup(option);
+ *detail = NULL;
+ }
+ else
+ {
+ char *alg;
+
+ alg = palloc((sep - option) + 1);
+ memcpy(alg, option, sep - option);
+ alg[sep - option] = '\0';
+
+ *algorithm = alg;
+ *detail = pstrdup(sep + 1);
+ }
+}
+#endif /* FRONTEND */
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 22836ca01a..4f56c9f464 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -956,27 +956,13 @@ parse_max_rate(char *src)
* at a later stage.
*/
static void
-parse_compress_options(char *option, char **algorithm, char **detail,
- CompressionLocation *locationres)
+backup_parse_compress_options(char *option, char **algorithm, char **detail,
+ CompressionLocation *locationres)
{
- char *sep;
- char *endp;
-
/*
- * Check whether the compression specification consists of a bare integer.
- *
- * If so, for backward compatibility, assume gzip.
+ * Strip off any "client-" or "server-" prefix, calculating the
+ * location.
*/
- (void) strtol(option, &endp, 10);
- if (*endp == '\0')
- {
- *locationres = COMPRESS_LOCATION_UNSPECIFIED;
- *algorithm = pstrdup("gzip");
- *detail = pstrdup(option);
- return;
- }
-
- /* Strip off any "client-" or "server-" prefix. */
if (strncmp(option, "server-", 7) == 0)
{
*locationres = COMPRESS_LOCATION_SERVER;
@@ -990,27 +976,8 @@ parse_compress_options(char *option, char **algorithm, char **detail,
else
*locationres = COMPRESS_LOCATION_UNSPECIFIED;
- /*
- * Check whether there is a compression detail following the algorithm
- * name.
- */
- sep = strchr(option, ':');
- if (sep == NULL)
- {
- *algorithm = pstrdup(option);
- *detail = NULL;
- }
- else
- {
- char *alg;
-
- alg = palloc((sep - option) + 1);
- memcpy(alg, option, sep - option);
- alg[sep - option] = '\0';
-
- *algorithm = alg;
- *detail = pstrdup(sep + 1);
- }
+ /* fallback to the common parsing for the algorithm and detail */
+ parse_compress_options(option, algorithm, detail);
}
/*
@@ -2411,8 +2378,8 @@ main(int argc, char **argv)
compressloc = COMPRESS_LOCATION_UNSPECIFIED;
break;
case 'Z':
- parse_compress_options(optarg, &compression_algorithm,
- &compression_detail, &compressloc);
+ backup_parse_compress_options(optarg, &compression_algorithm,
+ &compression_detail, &compressloc);
break;
case 'c':
if (pg_strcasecmp(optarg, "fast") == 0)
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index 63207ca025..c7a46b8a2a 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -57,8 +57,6 @@ static XLogRecPtr endpos = InvalidXLogRecPtr;
static void usage(void);
-static void parse_compress_options(char *option, char **algorithm,
- char **detail);
static DIR *get_destination_dir(char *dest_folder);
static void close_destination_dir(DIR *dest_dir, char *dest_folder);
static XLogRecPtr FindStreamingStart(uint32 *tli);
@@ -109,65 +107,6 @@ usage(void)
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
-/*
- * Basic parsing of a value specified for -Z/--compress
- *
- * The parsing consists of a METHOD:DETAIL string fed later on to a more
- * advanced routine in charge of proper validation checks. This only extracts
- * METHOD and DETAIL. If only an integer is found, the method is implied by
- * the value specified.
- */
-static void
-parse_compress_options(char *option, char **algorithm, char **detail)
-{
- char *sep;
- char *endp;
- long result;
-
- /*
- * Check whether the compression specification consists of a bare integer.
- *
- * For backward-compatibility, assume "none" if the integer found is zero
- * and "gzip" otherwise.
- */
- result = strtol(option, &endp, 10);
- if (*endp == '\0')
- {
- if (result == 0)
- {
- *algorithm = pstrdup("none");
- *detail = NULL;
- }
- else
- {
- *algorithm = pstrdup("gzip");
- *detail = pstrdup(option);
- }
- return;
- }
-
- /*
- * Check whether there is a compression detail following the algorithm
- * name.
- */
- sep = strchr(option, ':');
- if (sep == NULL)
- {
- *algorithm = pstrdup(option);
- *detail = NULL;
- }
- else
- {
- char *alg;
-
- alg = palloc((sep - option) + 1);
- memcpy(alg, option, sep - option);
- alg[sep - option] = '\0';
-
- *algorithm = alg;
- *detail = pstrdup(sep + 1);
- }
-}
/*
* Check if the filename looks like a WAL file, letting caller know if this
--
2.38.1
[application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-11-29 07:08 ` Michael Paquier <[email protected]>
1 sibling, 0 replies; 67+ messages in thread
From: Michael Paquier @ 2022-11-29 07:08 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Tue, Nov 29, 2022 at 03:19:17PM +0900, Michael Paquier wrote:
> Attached is an alternative that does not sacrifice the pluggability of
> the existing routines while allowing 0003~ to still use them (I don't
> really want to move around the checks on the supported build options
> now in parse_compress_specification(), that was hard enough to settle
> on this location). On top of that, pg_basebackup is able to cope with
> the case of --compress=0 already, enforcing "none" (BaseBackup could
> be simplified a bit more before StartLogStreamer). This refactoring
> shaves a little bit of code.
One thing that I forgot to mention is that this refactoring would
treat things like server-N, client-N as valid grammars (in this case N
takes precedence over an optional detail string), implying that N = 0
is "none" and N > 0 is gzip, so that makes for an extra grammar flavor
without impacting the existing ones. I am not sure that it is worth
documenting, still worth mentioning.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-11-29 12:10 ` [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
1 sibling, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-11-29 12:10 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Tuesday, November 29th, 2022 at 7:19 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Mon, Nov 28, 2022 at 04:32:43PM +0000, [email protected] wrote:
>
> > The focus of this version of this series is 0001 and 0002.
> >
> > Admittedly 0001 could be presented in a separate thread though given its size and
> > proximity to the topic, I present it here.
>
>
> I don't mind. This was a hole in meson.build, so nice catch! I have
> noticed a second defect with pg_verifybackup for all the commands, and
> applied both at the same time.
Thank you.
>
> > In an earlier review you spotted the similarity between pg_dump's and pg_receivewal's
> > parsing of compression options. However there exists a substantial difference in the
> > behaviour of the two programs; one treats the lack of support for the requested
> > algorithm as a fatal error, whereas the other does not. The existing functions in
> > common/compression.c do not account for the later. 0002 proposes an implementation
> > for this. It's usefulness is shown in 0003.
>
>
> In what does it matter? The logic in compression.c provides an error
> when looking at a spec or validating it, but the caller is free to
> consume it as it wants because this is shared between the frontend and
> the backend, and that includes consuming it as a warning rather than a
> ahrd failure. If we don't want to issue an error and force
> non-compression if attempting to use a compression method not
> supported in pg_dump, that's fine by me as a historical behavior, but
> I don't see why these routines have any need to be split more as
> proposed in 0002.
I understand. The reason for the change in the routines was because it was
impossible to distinguish a genuine parse error from a missing library in
parse_compress_specification(). If the zlib library is missing, then both
'--compress=gzip:garbage' and '--compress=gzip:7' would populate the
parse_error member of the struct and subsequent calls to
validate_compress_specification() would error out, although only one of
the two options is truly an error. Historically the code would fail on
invalid input regardless of whether the library was present or not.
> Saying that, I do agree that it would be nice to remove the
> duplication between the option parsing of pg_basebackup and
> pg_receivewal. Your patch is very close to that, actually, and it
> occured to me that if we move the check on "server-" and "client-" in
> pg_basebackup to be just before the integer-only check then we can
> consolidate the whole thing.
Great. I did notice the possible benefit but chose to not tread too far
off the necessary in my patch.
> Attached is an alternative that does not sacrifice the pluggability of
> the existing routines while allowing 0003~ to still use them (I don't
> really want to move around the checks on the supported build options
> now in parse_compress_specification(), that was hard enough to settle
> on this location).
Yeah, I thought that it would be a hard sell, hence an "earlier"
version.
The attached version 10, contains verbatim your proposed v9 as 0001.
Then 0002 is switching a bit the parsing order in pg_dump and will not
fail as described above on missing libraries. Now, it will first parse
the algorithm, discard it when unsupported, and only parse the rest of
the option if the algorithm is supported. Granted it is a bit 'uglier'
with the preprocessing blocks, yet it maintains most of the historic
behaviour without altering the common compression interfaces. Now, as
shown in 001_basic.pl, invalid detail will fail only if the algorithm
is supported.
> On top of that, pg_basebackup is able to cope with
> the case of --compress=0 already, enforcing "none" (BaseBackup could
> be simplified a bit more before StartLogStreamer). This refactoring
> shaves a little bit of code.
>
> > Please consider 0003-0005 as work in progress. They are differences from v7 yet they
> > may contain unaddressed comments for now.
>
>
> Okay.
Thank you. Please advice if is preferable to split 0002 in two parts.
I think not but I will happily do so if you think otherwise.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v10-0001-Make-the-pg_receivewal-compression-parsing-funct.patch (7.7K, ../../iKYD3Ryqsn88_UgYBiw3V803ISB5BJokmR2zsFtcedNVVeivIn_fqVv-oRRMqOpMiBFXsi72UYNMmuqMnEjjcQluTTpO-oxQWAVPeAlACnA=@pm.me/2-v10-0001-Make-the-pg_receivewal-compression-parsing-funct.patch)
download | inline diff:
From f9738acf3f6673447e11999ab02de703e2d01951 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Nov 2022 11:22:38 +0000
Subject: [PATCH v10 1/4] Make the pg_receivewal compression parsing function
common
Also and relax parsing errors in the helper functions and re-introduce those as
an independed function.
As it is shown in the rest of the patch series, there is a lot of duplication
between pg_dump's parsing of compression options and pg_receivewal's. Now the
core work is done in common. However pg_dump would not error out if the
requested compression algorithm is not supported by the build, whereas other
callers will error out. Also it seems a bit weird for only one of the parsing
functions for compressions to error out on missing support and that one to not
be the one responsible for identifying the compression algorithm.
A new function is added to test the support of the algorithm allowing the user
to tune the behaviour.
Authored by Micheal Paquier
---
src/bin/pg_basebackup/pg_basebackup.c | 49 ++++-----------------
src/bin/pg_basebackup/pg_receivewal.c | 61 --------------------------
src/common/compression.c | 63 +++++++++++++++++++++++++++
src/include/common/compression.h | 2 +
4 files changed, 73 insertions(+), 102 deletions(-)
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 22836ca01a..4f56c9f464 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -956,27 +956,13 @@ parse_max_rate(char *src)
* at a later stage.
*/
static void
-parse_compress_options(char *option, char **algorithm, char **detail,
- CompressionLocation *locationres)
+backup_parse_compress_options(char *option, char **algorithm, char **detail,
+ CompressionLocation *locationres)
{
- char *sep;
- char *endp;
-
/*
- * Check whether the compression specification consists of a bare integer.
- *
- * If so, for backward compatibility, assume gzip.
+ * Strip off any "client-" or "server-" prefix, calculating the
+ * location.
*/
- (void) strtol(option, &endp, 10);
- if (*endp == '\0')
- {
- *locationres = COMPRESS_LOCATION_UNSPECIFIED;
- *algorithm = pstrdup("gzip");
- *detail = pstrdup(option);
- return;
- }
-
- /* Strip off any "client-" or "server-" prefix. */
if (strncmp(option, "server-", 7) == 0)
{
*locationres = COMPRESS_LOCATION_SERVER;
@@ -990,27 +976,8 @@ parse_compress_options(char *option, char **algorithm, char **detail,
else
*locationres = COMPRESS_LOCATION_UNSPECIFIED;
- /*
- * Check whether there is a compression detail following the algorithm
- * name.
- */
- sep = strchr(option, ':');
- if (sep == NULL)
- {
- *algorithm = pstrdup(option);
- *detail = NULL;
- }
- else
- {
- char *alg;
-
- alg = palloc((sep - option) + 1);
- memcpy(alg, option, sep - option);
- alg[sep - option] = '\0';
-
- *algorithm = alg;
- *detail = pstrdup(sep + 1);
- }
+ /* fallback to the common parsing for the algorithm and detail */
+ parse_compress_options(option, algorithm, detail);
}
/*
@@ -2411,8 +2378,8 @@ main(int argc, char **argv)
compressloc = COMPRESS_LOCATION_UNSPECIFIED;
break;
case 'Z':
- parse_compress_options(optarg, &compression_algorithm,
- &compression_detail, &compressloc);
+ backup_parse_compress_options(optarg, &compression_algorithm,
+ &compression_detail, &compressloc);
break;
case 'c':
if (pg_strcasecmp(optarg, "fast") == 0)
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index 63207ca025..c7a46b8a2a 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -57,8 +57,6 @@ static XLogRecPtr endpos = InvalidXLogRecPtr;
static void usage(void);
-static void parse_compress_options(char *option, char **algorithm,
- char **detail);
static DIR *get_destination_dir(char *dest_folder);
static void close_destination_dir(DIR *dest_dir, char *dest_folder);
static XLogRecPtr FindStreamingStart(uint32 *tli);
@@ -109,65 +107,6 @@ usage(void)
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
-/*
- * Basic parsing of a value specified for -Z/--compress
- *
- * The parsing consists of a METHOD:DETAIL string fed later on to a more
- * advanced routine in charge of proper validation checks. This only extracts
- * METHOD and DETAIL. If only an integer is found, the method is implied by
- * the value specified.
- */
-static void
-parse_compress_options(char *option, char **algorithm, char **detail)
-{
- char *sep;
- char *endp;
- long result;
-
- /*
- * Check whether the compression specification consists of a bare integer.
- *
- * For backward-compatibility, assume "none" if the integer found is zero
- * and "gzip" otherwise.
- */
- result = strtol(option, &endp, 10);
- if (*endp == '\0')
- {
- if (result == 0)
- {
- *algorithm = pstrdup("none");
- *detail = NULL;
- }
- else
- {
- *algorithm = pstrdup("gzip");
- *detail = pstrdup(option);
- }
- return;
- }
-
- /*
- * Check whether there is a compression detail following the algorithm
- * name.
- */
- sep = strchr(option, ':');
- if (sep == NULL)
- {
- *algorithm = pstrdup(option);
- *detail = NULL;
- }
- else
- {
- char *alg;
-
- alg = palloc((sep - option) + 1);
- memcpy(alg, option, sep - option);
- alg[sep - option] = '\0';
-
- *algorithm = alg;
- *detail = pstrdup(sep + 1);
- }
-}
/*
* Check if the filename looks like a WAL file, letting caller know if this
diff --git a/src/common/compression.c b/src/common/compression.c
index df5b627834..5274ba5ba8 100644
--- a/src/common/compression.c
+++ b/src/common/compression.c
@@ -356,3 +356,66 @@ validate_compress_specification(pg_compress_specification *spec)
return NULL;
}
+
+#ifdef FRONTEND
+
+/*
+ * Basic parsing of a value specified through a command-line option, commonly
+ * -Z/--compress.
+ *
+ * The parsing consists of a METHOD:DETAIL string fed later to
+ * parse_compress_specification(). This only extracts METHOD and DETAIL.
+ * If only an integer is found, the method is implied by the value specified.
+ */
+void
+parse_compress_options(const char *option, char **algorithm, char **detail)
+{
+ char *sep;
+ char *endp;
+ long result;
+
+ /*
+ * Check whether the compression specification consists of a bare integer.
+ *
+ * For backward-compatibility, assume "none" if the integer found is zero
+ * and "gzip" otherwise.
+ */
+ result = strtol(option, &endp, 10);
+ if (*endp == '\0')
+ {
+ if (result == 0)
+ {
+ *algorithm = pstrdup("none");
+ *detail = NULL;
+ }
+ else
+ {
+ *algorithm = pstrdup("gzip");
+ *detail = pstrdup(option);
+ }
+ return;
+ }
+
+ /*
+ * Check whether there is a compression detail following the algorithm
+ * name.
+ */
+ sep = strchr(option, ':');
+ if (sep == NULL)
+ {
+ *algorithm = pstrdup(option);
+ *detail = NULL;
+ }
+ else
+ {
+ char *alg;
+
+ alg = palloc((sep - option) + 1);
+ memcpy(alg, option, sep - option);
+ alg[sep - option] = '\0';
+
+ *algorithm = alg;
+ *detail = pstrdup(sep + 1);
+ }
+}
+#endif /* FRONTEND */
diff --git a/src/include/common/compression.h b/src/include/common/compression.h
index 5d680058ed..46855b1a3b 100644
--- a/src/include/common/compression.h
+++ b/src/include/common/compression.h
@@ -33,6 +33,8 @@ typedef struct pg_compress_specification
char *parse_error; /* NULL if parsing was OK, else message */
} pg_compress_specification;
+extern void parse_compress_options(const char *option, char **algorithm,
+ char **detail);
extern bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm);
extern const char *get_compress_algorithm_name(pg_compress_algorithm algorithm);
--
2.34.1
[text/x-patch] v10-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (29.3K, ../../iKYD3Ryqsn88_UgYBiw3V803ISB5BJokmR2zsFtcedNVVeivIn_fqVv-oRRMqOpMiBFXsi72UYNMmuqMnEjjcQluTTpO-oxQWAVPeAlACnA=@pm.me/3-v10-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 8437ce812c45cc8477445e5c178e6c69fd45666d Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Nov 2022 11:23:00 +0000
Subject: [PATCH v10 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 41 +-
src/bin/pg_dump/compress_lz4.c | 593 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 9 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 12 +-
src/bin/pg_dump/t/001_basic.pl | 2 +-
src/bin/pg_dump/t/002_pg_dump.pl | 69 +++-
10 files changed, 742 insertions(+), 31 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 3fb8fdce81..84d3778c99 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 3065bd76fa..29e2352c31 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -129,6 +132,9 @@ AllocateCompressor(const pg_compress_specification compress_spec,
case PG_COMPRESSION_GZIP:
InitCompressorGzip(cs, compress_spec.level);
break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressorLZ4(cs, compress_spec.level);
+ break;
default:
pg_fatal("invalid compression method");
break;
@@ -179,6 +185,7 @@ free_keep_errno(void *p)
/*
* Compression None implementation
*/
+
static size_t
_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
@@ -314,6 +321,9 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
case PG_COMPRESSION_GZIP:
InitCompressGzip(CFH, compress_spec.level);
break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressLZ4(CFH, compress_spec.level);
+ break;
default:
pg_fatal("invalid compression method");
break;
@@ -326,12 +336,12 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -367,6 +377,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..6f4680c344
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,593 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..fbec9a508d
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, int compressionLevel);
+extern void InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 0c73a4707e..b27e92ffd0 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -83,7 +84,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 09e20fb97b..c9b053d572 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -394,6 +394,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2073,7 +2077,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2083,6 +2087,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3746,6 +3754,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
{
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 62c09e6ed4..d3feb062cd 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1274,20 +1274,25 @@ parse_compression(const char *opt, pg_compress_specification *compress_spec)
parse_compress_options(opt, &algorithm_str, &level_str);
if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
{
- pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ pg_log_error("invalid compression method: \"%s\" (gzip, lz4, none)",
algorithm_str);
return false;
}
/* Switch off unimplemented or unavailable compressions. */
if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
- compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ compress_spec->algorithm != PG_COMPRESSION_GZIP &&
+ compress_spec->algorithm != PG_COMPRESSION_LZ4)
supports_compression = false;
#ifndef HAVE_LIBZ
if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
#endif
+#ifndef USE_LZ4
+ if (compress_spec->algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
if (!supports_compression)
{
@@ -1310,6 +1315,9 @@ parse_compression(const char *opt, pg_compress_specification *compress_spec)
return false;
}
+ pg_free(algorithm_str);
+ pg_free(level_str);
+
return true;
}
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index f8d0b2fce5..2f7f389681 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,7 +122,7 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index d604558f03..0bec824836 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -116,6 +116,67 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=1', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4127,11 +4188,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
--
2.34.1
[text/x-patch] v10-0002-Prepare-pg_dump-for-additional-compression-metho.patch (50.6K, ../../iKYD3Ryqsn88_UgYBiw3V803ISB5BJokmR2zsFtcedNVVeivIn_fqVv-oRRMqOpMiBFXsi72UYNMmuqMnEjjcQluTTpO-oxQWAVPeAlACnA=@pm.me/4-v10-0002-Prepare-pg_dump-for-additional-compression-metho.patch)
download | inline diff:
From 516ea1bb70a4243965e231092ee460bb4ade311d Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Nov 2022 11:22:44 +0000
Subject: [PATCH v10 2/4] Prepare pg_dump for additional compression methods
This commmit does some of the heavy lifting required for additional compression
methods.
First it is teaching pg_dump.c about the definitions and interfaces found in
common/compression.h. Then it is propagating those throughout the code.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 431 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 20 +-
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 192 ++++++------
src/bin/pg_dump/pg_backup_archiver.h | 37 +--
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 99 ++++--
src/bin/pg_dump/t/001_basic.pl | 27 +-
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
12 files changed, 514 insertions(+), 362 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8b9d9f4cad..3fb8fdce81 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..4a8fc1e306 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -64,7 +68,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_algorithm compress_algorithm;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +78,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +94,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compress_algorithm = compress_spec.algorithm;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compress_spec.level);
#endif
return cs;
@@ -154,21 +128,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compress_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +156,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compress_algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +180,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compress_algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +410,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_algorithm compress_algorithm;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -452,21 +442,25 @@ cfp *
cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compress_spec = {0};
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compress_spec);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compress_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +473,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compress_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compress_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -509,60 +504,96 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_algorithm compress_algorithm, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compress_algorithm = compress_algorithm;
+
+ switch (compress_algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(path, -1, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(NULL, fd, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +603,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +665,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +698,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compress_algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..d6335fff02 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compress_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compress_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index e8b7898297..61c412c8cb 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compress_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index f39c0fa36f..304cc072ca 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +93,10 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +235,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +251,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compress_spec = {0};
+
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compress_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -354,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -383,16 +383,23 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ supports_compression = true;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -459,8 +466,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compress_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +746,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +976,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compress_spec.level = INT_MIN;
return opts;
}
@@ -1115,23 +1124,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compress_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compress_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compress_spec.algorithm));
switch (AH->format)
{
@@ -1485,60 +1499,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compress_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compress_spec);
if (!AH->OF)
{
@@ -1549,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1699,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2198,10 +2173,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2249,14 +2226,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compress_spec = compress_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2264,7 +2241,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3669,7 +3646,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compress_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3740,21 +3717,26 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compress_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compress_spec.level = ReadInt(AH);
+
+ if (AH->compress_spec.level != 0)
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 42687c4ec8..d2930949ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compress_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index a0a55a1edd..6a2112c45f 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 798182b6f7..7d2cddbb2c 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 402b93c610..d773c291c8 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +801,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +888,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index da427f4d4a..62c09e6ed4 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -54,8 +54,10 @@
#include "catalog/pg_subscription.h"
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
+#include "common/compression.h"
#include "common/connect.h"
#include "common/relpath.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -164,6 +166,8 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression(const char *opt,
+ pg_compress_specification *compress_spec);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -340,8 +344,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ pg_compress_specification compress_spec = {0};
+ bool user_compression_defined = false;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -561,10 +566,10 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression(optarg, &compress_spec))
exit_nicely(1);
+ user_compression_defined = true;
break;
case 0:
@@ -687,23 +692,20 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /*
+ * Custom and directory formats are compressed by default (zlib), others
+ * not
+ */
+ if (user_compression_defined == false)
{
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, &compress_spec);
#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ parse_compress_specification(PG_COMPRESSION_GZIP, NULL,
+ &compress_spec);
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -716,8 +718,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compress_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -948,10 +950,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compress_spec = compress_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -998,7 +997,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1258,6 +1258,61 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+/*
+ * Interprets and validates a compression option using the common compression
+ * parsing functions. If the requested compression is not available then the
+ * archives are uncompressed.
+ */
+static bool
+parse_compression(const char *opt, pg_compress_specification *compress_spec)
+{
+ char *algorithm_str = NULL;
+ char *level_str = NULL;
+ char *validation_error = NULL;
+ bool supports_compression = true;
+
+ parse_compress_options(opt, &algorithm_str, &level_str);
+ if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
+ {
+ pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ algorithm_str);
+ return false;
+ }
+
+ /* Switch off unimplemented or unavailable compressions. */
+ if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
+ compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ supports_compression = false;
+
+#ifndef HAVE_LIBZ
+ if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, compress_spec);
+
+ pg_free(algorithm_str);
+ pg_free(level_str);
+
+ return true;
+ }
+
+ /* Parse and validate the rest of the options */
+ parse_compress_specification(compress_spec->algorithm, level_str,
+ compress_spec);
+ validation_error = validate_compress_specification(compress_spec);
+ if (validation_error)
+ {
+ pg_log_error("invalid compression specification: %s", validation_error);
+ return false;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..f8d0b2fce5 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,16 +121,32 @@ command_fails_like(
'pg_restore: cannot specify both --single-transaction and multiple jobs');
command_fails_like(
- [ 'pg_dump', '-Z', '-1' ],
- qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
- 'pg_dump: -Z/--compress must be in range');
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level');
+
if (check_pg_config("#define HAVE_LIBZ 1"))
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '15' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
else
{
@@ -139,6 +155,11 @@ else
[ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: compression not available in this installation');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ],
+ qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index fe53ed0f89..d604558f03 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
--
2.34.1
[text/x-patch] v10-0003-Introduce-Compressor-API-in-pg_dump.patch (53.9K, ../../iKYD3Ryqsn88_UgYBiw3V803ISB5BJokmR2zsFtcedNVVeivIn_fqVv-oRRMqOpMiBFXsi72UYNMmuqMnEjjcQluTTpO-oxQWAVPeAlACnA=@pm.me/5-v10-0003-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From 8285412481539bb6649616b24df5191a9423f3c1 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Nov 2022 11:22:48 +0000
Subject: [PATCH v10 3/4] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives need to now store the compression method in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 390 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 9 +
src/bin/pg_dump/compress_io.c | 817 ++++++--------------------
src/bin/pg_dump/compress_io.h | 69 ++-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 93 +--
src/bin/pg_dump/pg_backup_archiver.h | 4 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 85 +--
10 files changed, 765 insertions(+), 727 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..bc6d1abc77
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,390 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ int compressionLevel;
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, gzipcs->compressionLevel) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+typedef struct GzipData
+{
+ gzFile fp;
+ int compressionLevel;
+} GzipData;
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ size_t ret;
+
+ ret = gzread(gd->fp, ptr, size);
+ if (ret != size && !gzeof(gd->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gd->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzwrite(gd->fp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gd->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gd->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzgets(gd->fp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gd->fp);
+
+ save_errno = errno;
+ free(gd);
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzeof(gd->fp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gd->fp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ char mode_compression[32];
+
+ if (gd->compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, gd->compressionLevel);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gd->fp = gzdopen(dup(fd), mode_compression);
+ else
+ gd->fp = gzopen(path, mode_compression);
+
+ if (gd->fp == NULL)
+ return 1;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ GzipData *gd;
+
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ gd = pg_malloc0(sizeof(GzipData));
+ gd->compressionLevel = compressionLevel;
+
+ CFH->private = gd;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..ab0362c1f3
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, int compressionLevel);
+extern void InitCompressGzip(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 4a8fc1e306..3065bd76fa 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,113 +68,73 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_algorithm compress_algorithm;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+ free(buf);
+}
+
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compress_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compress_algorithm = compress_spec.algorithm;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, compress_spec.level);
-#endif
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
- ReadFunc readF)
-{
switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressorGzip(cs, compress_spec.level);
break;
default:
pg_fatal("invalid compression method");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compress_algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ return cs;
}
/*
@@ -180,243 +143,28 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compress_algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- free(cs);
- break;
-
- default:
- pg_fatal("invalid compression method");
- break;
- }
-}
-
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
- */
-
-static void
-InitCompressorZlib(CompressorState *cs, int level)
-{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
-}
-
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
-{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
-{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
-
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
-
- if (res == Z_STREAM_END)
- break;
- }
-}
-
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
-
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
-{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
-}
-#endif /* HAVE_LIBZ */
-
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
-{
- size_t cnt;
- char *buf;
- size_t buflen;
-
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
-
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
-
- free(buf);
-}
-
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->writeF(AH, data, dLen);
-}
-
-
/*----------------------
* Compressed stream API
*----------------------
*/
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- pg_compress_algorithm compress_algorithm;
- void *fp;
-};
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
+}
/* free() without changing errno; useful in several places below */
static void
@@ -429,392 +177,219 @@ free_keep_errno(void *p)
}
/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
+ * Compression None implementation
*/
-cfp *
-cfopen_read(const char *path, const char *mode)
+static size_t
+_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp;
- pg_compress_specification compress_spec = {0};
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, compress_spec);
- else
-#endif
- {
- compress_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compress_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
-
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
-}
+ if (size == 0)
+ return 0;
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
-{
- cfp *fp;
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
- if (compress_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compress_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ return ret;
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_algorithm compress_algorithm, int compressionLevel)
+static size_t
+_write(const void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
-
- fp->compress_algorithm = compress_algorithm;
-
- switch (compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compressionLevel != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compressionLevel);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
-
- return fp;
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
+static const char *
+_get_error(CompressFileHandle * CFH)
{
- return cfopen_internal(path, -1, mode,
- compress_spec.algorithm,
- compress_spec.level);
+ return strerror(errno);
}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compress_spec)
+static char *
+_gets(char *ptr, int size, CompressFileHandle * CFH)
{
- return cfopen_internal(NULL, fd, mode,
- compress_spec.algorithm,
- compress_spec.level);
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-int
-cfread(void *ptr, int size, cfp *fp)
+static int
+_getc(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret;
- if (size == 0)
- return 0;
-
- switch (fp->compress_algorithm)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
-
- default:
- pg_fatal("invalid compression method");
- break;
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
return ret;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+static int
+_close(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret = 0;
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ CFH->private = NULL;
+
+ if (fp)
+ ret = fclose(fp);
return ret;
}
-int
-cfgetc(cfp *fp)
+static int
+_eof(CompressFileHandle * CFH)
{
- int ret;
+ return feof((FILE *) CFH->private);
+}
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
+static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ Assert(CFH->private == NULL);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- return ret;
+ if (CFH->private == NULL)
+ return 1;
+
+ return 0;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+static int
+_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
{
- char *ret;
+ Assert(CFH->private == NULL);
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ return 0;
+}
- return ret;
+static void
+InitCompressNone(CompressFileHandle * CFH)
+{
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-int
-cfclose(cfp *fp)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compress_spec)
{
- int ret;
+ CompressFileHandle *CFH;
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
- switch (fp->compress_algorithm)
+ switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
+ InitCompressNone(CFH);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressGzip(CFH, compress_spec.level);
break;
default:
pg_fatal("invalid compression method");
break;
}
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
-int
-cfeof(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret;
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compress_spec = {0};
- switch (fp->compress_algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- default:
- pg_fatal("invalid compression method");
- break;
- }
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- return ret;
-}
+ fname = strdup(path);
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
+ if (hasSuffix(fname, ".gz"))
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
+ bool exists;
+
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("not built with zlib support");
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
}
- return strerror(errno);
+ CFH = InitCompressFileHandle(compress_spec);
+ if (CFH->open(fname, -1, mode, CFH))
+ {
+ free_keep_errno(CFH);
+ CFH = NULL;
+ }
+ free_keep_errno(fname);
+
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle * CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d6335fff02..a986f5e6ee 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,61 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compress_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle * cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle * CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle * CFH);
+ int (*getc) (CompressFileHandle * CFH);
+ int (*eof) (CompressFileHandle * CFH);
+ int (*close) (CompressFileHandle * CFH);
+ const char *(*get_error) (CompressFileHandle * CFH);
+
+ void *private;
+};
+
-typedef struct cfp cfp;
+extern CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compress_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compress_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle * InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle * CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index d96e566846..0c73a4707e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 304cc072ca..09e20fb97b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle * SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1126,7 +1126,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compress_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1502,6 +1502,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1525,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compress_spec);
- else
- AH->OF = cfopen(filename, mode, compress_spec);
+ CFH = InitCompressFileHandle(compress_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1689,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2035,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2077,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2180,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2236,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,7 +3652,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compress_spec.level);
+ AH->WriteBytePtr(AH, AH->compress_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3718,7 +3724,9 @@ ReadHead(ArchiveHandle *AH)
AH->format, fmt);
AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compress_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
AH->compress_spec.level = AH->ReadBytePtr(AH);
@@ -3731,11 +3739,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
-
+ if (unsupported)
+ {
+ pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ }
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index d2930949ab..bb7fad2af1 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,12 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 6a2112c45f..49ec0e3816 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compress_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7d2cddbb2c..e1ce2f393b 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,9 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
+ CompressFileHandle *dataFH; /* currently open data file */
- cfp *blobsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *blobsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +198,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +218,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compress_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +346,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +371,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +386,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +435,7 @@ _LoadBlobs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +443,14 @@ _LoadBlobs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->blobsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the blobs TOC file line-by-line, and process each blob */
- while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char blobfname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +465,11 @@ _LoadBlobs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreBlob(AH, oid);
}
- if (!cfeof(ctx->blobsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +489,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +514,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +527,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +549,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +578,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +589,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compress_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +603,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +654,8 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
/* The blob TOC file is never compressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
- if (ctx->blobsTocFH == NULL)
+ ctx->blobsTocFH = InitCompressFileHandle(compress_spec);
+ if (ctx->blobsTocFH->open_write(fname, "ab", ctx->blobsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +672,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,17 +686,18 @@ static void
_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->blobsTocFH;
char buf[50];
int len;
/* Close the BLOB data file itself */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the blob in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->blobsTocFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
pg_fatal("could not write to blobs TOC file");
}
@@ -706,7 +711,7 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close blobs TOC file: %m");
ctx->blobsTocFH = NULL;
}
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-11-30 00:50 ` Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-11-30 00:50 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Tue, Nov 29, 2022 at 12:10:46PM +0000, [email protected] wrote:
> Thank you. Please advice if is preferable to split 0002 in two parts.
> I think not but I will happily do so if you think otherwise.
This one makes me curious. What kind of split are you talking about?
If it makes the code review and the git history cleaner and easier, I
am usually a lot in favor of such incremental changes. As far as I
can see, there is the switch from the compression integer to
compression specification as one thing. The second thing is the
refactoring of cfclose() and these routines, paving the way for 0003.
Hmm, it may be cleaner to move the switch to the compression spec in
one patch, and move the logic around cfclose() to its own, paving the
way to 0003.
By the way, I think that this 0002 should drop all the default clauses
in the switches for the compression method so as we'd catch any
missing code paths with compiler warnings if a new compression method
is added in the future.
Anyway, I have applied 0001, adding you as a primary author because
you did most of it with only tweaks from me for pg_basebackup. The
docs of pg_basebackup have been amended to mention the slight change
in grammar, affecting the case where we do not have a detail string.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../Y4ao2viUjcO%[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-11-30 17:11 ` [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-11-30 17:11 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Wednesday, November 30th, 2022 at 1:50 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Tue, Nov 29, 2022 at 12:10:46PM +0000, [email protected] wrote:
>
> > Thank you. Please advice if is preferable to split 0002 in two parts.
> > I think not but I will happily do so if you think otherwise.
>
>
> This one makes me curious. What kind of split are you talking about?
> If it makes the code review and the git history cleaner and easier, I
> am usually a lot in favor of such incremental changes. As far as I
> can see, there is the switch from the compression integer to
> compression specification as one thing. The second thing is the
> refactoring of cfclose() and these routines, paving the way for 0003.
> Hmm, it may be cleaner to move the switch to the compression spec in
> one patch, and move the logic around cfclose() to its own, paving the
> way to 0003.
Fair enough. The atteched v11 does that. 0001 introduces compression
specification and is using it throughout. 0002 paves the way to the
new interface by homogenizing the use of cfp. 0003 introduces the new
API and stores the compression algorithm in the custom format header
instead of the compression level integer. Finally 0004 adds support for
LZ4.
Besides the version bump in 0003 which can possibly be split out and
as an independent and earlier step, I think that the patchset consists
of coherent units.
> By the way, I think that this 0002 should drop all the default clauses
> in the switches for the compression method so as we'd catch any
> missing code paths with compiler warnings if a new compression method
> is added in the future.
Sure.
> Anyway, I have applied 0001, adding you as a primary author because
> you did most of it with only tweaks from me for pg_basebackup. The
> docs of pg_basebackup have been amended to mention the slight change
> in grammar, affecting the case where we do not have a detail string.
Very kind of you, thank you.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v11-0002-Prepare-pg_dump-internals-for-additional-compres.patch (20.1K, ../../1j_ONBQqU3lQW7OqwbAFKbgcqjQosikcp7VPINqegLrHMCMGtFcXAuyjmW__VeviKyCbKTtucxFyPFUJ7sfqeOf4O90JEuUIe99Z5v7sdOg=@pm.me/2-v11-0002-Prepare-pg_dump-internals-for-additional-compres.patch)
download | inline diff:
From 18c8ba5e115c8f3386ac5421d1cc595ef8ed2a62 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 30 Nov 2022 14:07:39 +0000
Subject: [PATCH v11 2/4] Prepare pg_dump internals for additional compression
methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about it and is using it throughout.
---
src/bin/pg_dump/compress_io.c | 363 ++++++++++++++++++---------
src/bin/pg_dump/pg_backup_archiver.c | 130 ++++------
src/bin/pg_dump/pg_backup_archiver.h | 27 +-
3 files changed, 297 insertions(+), 223 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 2c9d730fce..83b478bc63 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -127,15 +131,23 @@ void
ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
ReadFunc readF)
{
- if (compress_spec.algorithm == PG_COMPRESSION_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ switch (compress_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -172,11 +184,24 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compress_spec.algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->compress_spec.algorithm == PG_COMPRESSION_GZIP)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -390,10 +415,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_specification compress_spec;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -486,127 +509,195 @@ cfopen_write(const char *path, const char *mode,
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_specification compress_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ fp->compress_spec = compress_spec;
+
+ switch (compress_spec.algorithm)
{
-#ifdef HAVE_LIBZ
- if (compress_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compress_spec.level);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compress_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compress_spec.level);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(path, -1, mode, compress_spec);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(NULL, fd, mode, compress_spec);
+}
int
cfread(void *ptr, int size, cfp *fp)
{
- int ret;
+ int ret = 0;
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_spec.algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compress_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfgetc(cfp *fp)
{
- int ret;
+ int ret = 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_spec.algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -615,65 +706,113 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret = NULL;
+
+ switch (fp->compress_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret = 0;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compress_spec.algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compress_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compress_spec.algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 79347d387b..92a160b67e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -101,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -277,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -362,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -391,17 +383,24 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->compress_spec.algorithm == PG_COMPRESSION_GZIP &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -979,7 +978,7 @@ NewRestoreOptions(void)
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
- opts->compress_spec.level = 0;
+ opts->compress_spec.level = INT_MIN;
return opts;
}
@@ -1128,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compress_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
@@ -1504,58 +1503,32 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compress_spec.level);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compress_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compress_spec);
if (!AH->OF)
{
@@ -1566,33 +1539,24 @@ SetOutput(ArchiveHandle *AH, const char *filename,
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1716,22 +1680,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2220,6 +2179,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2273,8 +2233,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index d58b96b2dc..d2930949ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
--
2.34.1
[text/x-patch] v11-0003-Introduce-Compressor-API-in-pg_dump.patch (54.5K, ../../1j_ONBQqU3lQW7OqwbAFKbgcqjQosikcp7VPINqegLrHMCMGtFcXAuyjmW__VeviKyCbKTtucxFyPFUJ7sfqeOf4O90JEuUIe99Z5v7sdOg=@pm.me/3-v11-0003-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From 0c8e498070fa30f3f05657cd769df3053a39e1d8 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Nov 2022 11:22:48 +0000
Subject: [PATCH v11 3/4] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives now need to store the compression algorithm in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 390 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 9 +
src/bin/pg_dump/compress_io.c | 829 ++++++--------------------
src/bin/pg_dump/compress_io.h | 69 ++-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 93 +--
src/bin/pg_dump/pg_backup_archiver.h | 4 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 85 +--
10 files changed, 766 insertions(+), 738 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..bc6d1abc77
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,390 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ int compressionLevel;
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, gzipcs->compressionLevel) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+typedef struct GzipData
+{
+ gzFile fp;
+ int compressionLevel;
+} GzipData;
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ size_t ret;
+
+ ret = gzread(gd->fp, ptr, size);
+ if (ret != size && !gzeof(gd->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gd->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzwrite(gd->fp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gd->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gd->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzgets(gd->fp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gd->fp);
+
+ save_errno = errno;
+ free(gd);
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzeof(gd->fp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gd->fp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ char mode_compression[32];
+
+ if (gd->compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, gd->compressionLevel);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gd->fp = gzdopen(dup(fd), mode_compression);
+ else
+ gd->fp = gzopen(path, mode_compression);
+
+ if (gd->fp == NULL)
+ return 1;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ GzipData *gd;
+
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ gd = pg_malloc0(sizeof(GzipData));
+ gd->compressionLevel = compressionLevel;
+
+ CFH->private = gd;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..ab0362c1f3
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, int compressionLevel);
+extern void InitCompressGzip(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 83b478bc63..56a07e309b 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,83 +68,66 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_specification compress_spec;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
+
+ free(buf);
+}
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compress_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compress_spec = compress_spec;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compress_spec.algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, cs->compress_spec.level);
-#endif
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
- ReadFunc readF)
-{
switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressorGzip(cs, compress_spec.level);
break;
case PG_COMPRESSION_LZ4:
/* fallthrough */
@@ -149,33 +135,8 @@ ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
pg_fatal("invalid compression method");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compress_spec.algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ return cs;
}
/*
@@ -184,244 +145,28 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compress_spec.algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- free(cs);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-}
-
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
- */
-
-static void
-InitCompressorZlib(CompressorState *cs, int level)
-{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ cs->end(AH, cs);
+ pg_free(cs);
}
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
-{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
-}
-
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
-{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
-
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
-
- if (res == Z_STREAM_END)
- break;
- }
-}
-
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
-
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
-{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
-}
-#endif /* HAVE_LIBZ */
-
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
-{
- size_t cnt;
- char *buf;
- size_t buflen;
-
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
-
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
-
- free(buf);
-}
-
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->writeF(AH, data, dLen);
-}
-
-
/*----------------------
* Compressed stream API
*----------------------
*/
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- pg_compress_specification compress_spec;
- void *fp;
-};
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
+}
/* free() without changing errno; useful in several places below */
static void
@@ -434,328 +179,142 @@ free_keep_errno(void *p)
}
/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
+ * Compression None implementation
*/
-cfp *
-cfopen_read(const char *path, const char *mode)
+static size_t
+_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp;
- pg_compress_specification compress_spec = {0};
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, compress_spec);
- else
-#endif
- {
- compress_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compress_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
+ if (size == 0)
+ return 0;
- compress_spec.algorithm = PG_COMPRESSION_GZIP;
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
-}
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
-{
- cfp *fp;
-
- if (compress_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compress_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compress_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ return ret;
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_specification compress_spec)
+static size_t
+_write(const void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
-
- fp->compress_spec = compress_spec;
-
- switch (compress_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compress_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compress_spec.level);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return fp;
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec)
+static const char *
+_get_error(CompressFileHandle * CFH)
{
- return cfopen_internal(path, -1, mode, compress_spec);
+ return strerror(errno);
}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compress_spec)
+static char *
+_gets(char *ptr, int size, CompressFileHandle * CFH)
{
- return cfopen_internal(NULL, fd, mode, compress_spec);
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-int
-cfread(void *ptr, int size, cfp *fp)
+static int
+_getc(CompressFileHandle * CFH)
{
- int ret = 0;
-
- if (size == 0)
- return 0;
+ FILE *fp = (FILE *) CFH->private;
+ int ret;
- switch (fp->compress_spec.algorithm)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
return ret;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+static int
+_close(CompressFileHandle * CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret = 0;
- switch (fp->compress_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ CFH->private = NULL;
+
+ if (fp)
+ ret = fclose(fp);
return ret;
}
-int
-cfgetc(cfp *fp)
+static int
+_eof(CompressFileHandle * CFH)
{
- int ret = 0;
+ return feof((FILE *) CFH->private);
+}
- switch (fp->compress_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
+static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ Assert(CFH->private == NULL);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- return ret;
+ if (CFH->private == NULL)
+ return 1;
+
+ return 0;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+static int
+_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
{
- char *ret = NULL;
+ Assert(CFH->private == NULL);
- switch (fp->compress_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ return 0;
+}
- return ret;
+static void
+InitCompressNone(CompressFileHandle * CFH)
+{
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-int
-cfclose(cfp *fp)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compress_spec)
{
- int ret = 0;
+ CompressFileHandle *CFH;
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
- switch (fp->compress_spec.algorithm)
+ switch (compress_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
+ InitCompressNone(CFH);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressGzip(CFH, compress_spec.level);
break;
case PG_COMPRESSION_LZ4:
/* fallthrough */
@@ -764,71 +323,77 @@ cfclose(cfp *fp)
break;
}
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
-int
-cfeof(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret = 0;
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compress_spec = {0};
- switch (fp->compress_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- return ret;
-}
+ fname = strdup(path);
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ if (hasSuffix(fname, ".gz"))
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
+ bool exists;
+
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("not built with zlib support");
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
}
- return strerror(errno);
+ CFH = InitCompressFileHandle(compress_spec);
+ if (CFH->open(fname, -1, mode, CFH))
+ {
+ free_keep_errno(CFH);
+ CFH = NULL;
+ }
+ free_keep_errno(fname);
+
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle * CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d6335fff02..a986f5e6ee 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,61 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compress_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle * cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle * CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle * CFH);
+ int (*getc) (CompressFileHandle * CFH);
+ int (*eof) (CompressFileHandle * CFH);
+ int (*close) (CompressFileHandle * CFH);
+ const char *(*get_error) (CompressFileHandle * CFH);
+
+ void *private;
+};
+
-typedef struct cfp cfp;
+extern CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compress_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compress_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compress_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle * InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle * CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index d96e566846..0c73a4707e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 92a160b67e..248646143d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle * SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1127,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compress_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1503,6 +1503,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compress_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1525,33 +1526,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compress_spec);
- else
- AH->OF = cfopen(filename, mode, compress_spec);
+ CFH = InitCompressFileHandle(compress_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1690,7 +1690,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2032,6 +2036,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2062,26 +2078,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2179,6 +2181,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2234,7 +2237,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3647,7 +3653,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compress_spec.level);
+ AH->WriteBytePtr(AH, AH->compress_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3719,7 +3725,9 @@ ReadHead(ArchiveHandle *AH)
AH->format, fmt);
AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compress_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
AH->compress_spec.level = AH->ReadBytePtr(AH);
@@ -3732,11 +3740,20 @@ ReadHead(ArchiveHandle *AH)
else
AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
-
+ if (unsupported)
+ {
+ pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ }
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index d2930949ab..bb7fad2af1 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,12 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 6a2112c45f..49ec0e3816 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compress_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 7d2cddbb2c..e1ce2f393b 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,9 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
+ CompressFileHandle *dataFH; /* currently open data file */
- cfp *blobsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *blobsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +198,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +218,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compress_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +346,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +371,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +386,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +435,7 @@ _LoadBlobs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +443,14 @@ _LoadBlobs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->blobsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the blobs TOC file line-by-line, and process each blob */
- while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char blobfname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +465,11 @@ _LoadBlobs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreBlob(AH, oid);
}
- if (!cfeof(ctx->blobsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +489,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +514,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +527,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +549,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +578,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +589,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compress_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +603,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +654,8 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
/* The blob TOC file is never compressed */
compress_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
- if (ctx->blobsTocFH == NULL)
+ ctx->blobsTocFH = InitCompressFileHandle(compress_spec);
+ if (ctx->blobsTocFH->open_write(fname, "ab", ctx->blobsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +672,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compress_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,17 +686,18 @@ static void
_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->blobsTocFH;
char buf[50];
int len;
/* Close the BLOB data file itself */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the blob in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->blobsTocFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
pg_fatal("could not write to blobs TOC file");
}
@@ -706,7 +711,7 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close blobs TOC file: %m");
ctx->blobsTocFH = NULL;
}
--
2.34.1
[text/x-patch] v11-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (29.5K, ../../1j_ONBQqU3lQW7OqwbAFKbgcqjQosikcp7VPINqegLrHMCMGtFcXAuyjmW__VeviKyCbKTtucxFyPFUJ7sfqeOf4O90JEuUIe99Z5v7sdOg=@pm.me/4-v11-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From ca4de793a6ae3ac0bd34e68809a53b8d2b1abb4f Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 29 Nov 2022 11:23:00 +0000
Subject: [PATCH v11 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 23 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 43 +-
src/bin/pg_dump/compress_lz4.c | 601 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 9 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 12 +-
src/bin/pg_dump/t/001_basic.pl | 2 +-
src/bin/pg_dump/t/002_pg_dump.pl | 69 ++-
10 files changed, 749 insertions(+), 34 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 3fb8fdce81..84d3778c99 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,12 +653,12 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression. A compression level can
- be optionally specified, by appending the level number after a colon
- (<literal>:</literal>). If no level is specified, the default compression
- level will be used for the specified method. If only a level is
- specified without mentioning a method, <literal>gzip</literal> compression
- will be used.
+ <literal>lz4</literal> or <literal>none</literal> for no compression. A
+ compression level can be optionally specified, by appending the level
+ number after a colon (<literal>:</literal>). If no level is specified,
+ the default compression level will be used for the specified method. If
+ only a level is specified without mentioning a method,
+ <literal>gzip</literal> compression willbe used.
</para>
<para>
@@ -665,8 +666,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 56a07e309b..731c0913df 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -130,8 +133,9 @@ AllocateCompressor(const pg_compress_specification compress_spec,
InitCompressorGzip(cs, compress_spec.level);
break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
+ InitCompressorLZ4(cs, compress_spec.level);
+ break;
+ default:
pg_fatal("invalid compression method");
break;
}
@@ -181,6 +185,7 @@ free_keep_errno(void *p)
/*
* Compression None implementation
*/
+
static size_t
_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
@@ -317,7 +322,8 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
InitCompressGzip(CFH, compress_spec.level);
break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ InitCompressLZ4(CFH, compress_spec.level);
+ break;
case PG_COMPRESSION_ZSTD:
pg_fatal("invalid compression method");
break;
@@ -330,12 +336,12 @@ InitCompressFileHandle(const pg_compress_specification compress_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -371,6 +377,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compress_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..8f93f05e87
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,601 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*
+ * LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
+ * Redefine it for installations with a lesser version.
+ */
+#ifndef LZ4F_HEADER_SIZE_MAX
+#define LZ4F_HEADER_SIZE_MAX 32
+#endif
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..fbec9a508d
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, int compressionLevel);
+extern void InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 0c73a4707e..b27e92ffd0 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -83,7 +84,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 248646143d..51e51748f1 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -395,6 +395,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2074,7 +2078,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2084,6 +2088,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3747,6 +3755,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
{
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 4830983c86..78016ba1d2 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -1272,20 +1272,25 @@ parse_compression(const char *opt, pg_compress_specification *compress_spec)
parse_compress_options(opt, &algorithm_str, &level_str);
if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
{
- pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ pg_log_error("invalid compression method: \"%s\" (gzip, lz4, none)",
algorithm_str);
return false;
}
/* Switch off unimplemented or unavailable compressions. */
if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
- compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ compress_spec->algorithm != PG_COMPRESSION_GZIP &&
+ compress_spec->algorithm != PG_COMPRESSION_LZ4)
supports_compression = false;
#ifndef HAVE_LIBZ
if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
#endif
+#ifndef USE_LZ4
+ if (compress_spec->algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
+#endif
if (!supports_compression)
{
@@ -1308,6 +1313,9 @@ parse_compression(const char *opt, pg_compress_specification *compress_spec)
return false;
}
+ pg_free(algorithm_str);
+ pg_free(level_str);
+
return true;
}
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index f8d0b2fce5..2f7f389681 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -122,7 +122,7 @@ command_fails_like(
command_fails_like(
[ 'pg_dump', '--compress', 'garbage' ],
- qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, lz4, none)\E/,
'pg_dump: invalid --compress');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index d604558f03..0bec824836 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -116,6 +116,67 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=1', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4127,11 +4188,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
--
2.34.1
[text/x-patch] v11-0001-Teach-pg_dump-about-compress_spec-and-use-it-thr.patch (35.4K, ../../1j_ONBQqU3lQW7OqwbAFKbgcqjQosikcp7VPINqegLrHMCMGtFcXAuyjmW__VeviKyCbKTtucxFyPFUJ7sfqeOf4O90JEuUIe99Z5v7sdOg=@pm.me/5-v11-0001-Teach-pg_dump-about-compress_spec-and-use-it-thr.patch)
download | inline diff:
From 1669812c3177a0c54e36d892676a45a51232096a Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 30 Nov 2022 12:58:03 +0000
Subject: [PATCH v11 1/4] Teach pg_dump about compress_spec and use it
throughout.
Align pg_dump with the rest of the binaries which use common compression. It is
teaching pg_dump.c about the common compression definitions and interfaces. Then
it propagates those throughout the code.
---
doc/src/sgml/ref/pg_dump.sgml | 30 ++++++--
src/bin/pg_dump/compress_io.c | 102 ++++++++++----------------
src/bin/pg_dump/compress_io.h | 20 ++---
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 71 ++++++++++++------
src/bin/pg_dump/pg_backup_archiver.h | 10 +--
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 +++-
src/bin/pg_dump/pg_backup_tar.c | 11 ++-
src/bin/pg_dump/pg_dump.c | 97 ++++++++++++++++++------
src/bin/pg_dump/t/001_basic.pl | 27 ++++++-
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
12 files changed, 243 insertions(+), 153 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8b9d9f4cad..3fb8fdce81 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..2c9d730fce 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_specification compress_spec;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compress_spec = compress_spec;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, cs->compress_spec.level);
#endif
return cs;
@@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
+ if (compress_spec.algorithm == PG_COMPRESSION_NONE)
ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
{
#ifdef HAVE_LIBZ
ReadDataFromArchiveZlib(AH, readF);
@@ -179,18 +146,23 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compress_spec.algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -201,7 +173,7 @@ void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
+ if (cs->compress_spec.algorithm == PG_COMPRESSION_GZIP)
EndCompressorZlib(AH, cs);
#endif
free(cs);
@@ -452,21 +424,25 @@ cfp *
cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compress_spec = {0};
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compress_spec);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compress_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +455,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compress_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compress_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -515,20 +492,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ if (compress_spec.level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compress_spec.level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..d6335fff02 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compress_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compress_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index e8b7898297..61c412c8cb 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compress_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index f39c0fa36f..79347d387b 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +99,8 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -239,12 +241,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +257,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compress_spec = {0};
+
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compress_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -384,7 +392,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +468,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compress_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +748,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +978,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compress_spec.level = 0;
return opts;
}
@@ -1115,23 +1126,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compress_spec = {0};
teSection curSection;
OutputContext sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compress_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compress_spec.algorithm));
switch (AH->format)
{
@@ -1485,7 +1501,8 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec)
{
int fn;
@@ -1508,12 +1525,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compress_spec.level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2198,7 +2215,8 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2249,7 +2267,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compress_spec = compress_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2264,7 +2282,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3669,7 +3687,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compress_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3740,21 +3758,26 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compress_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compress_spec.level = ReadInt(AH);
+
+ if (AH->compress_spec.level != 0)
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 42687c4ec8..d58b96b2dc 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -331,14 +331,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compress_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index a0a55a1edd..6a2112c45f 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 798182b6f7..7d2cddbb2c 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 402b93c610..87bfed76fd 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -194,7 +194,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +328,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +383,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +401,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +800,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +887,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index da427f4d4a..4830983c86 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -164,6 +164,8 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression(const char *opt,
+ pg_compress_specification *compress_spec);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -340,8 +342,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ pg_compress_specification compress_spec = {0};
+ bool user_compression_defined = false;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -561,10 +564,10 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression(optarg, &compress_spec))
exit_nicely(1);
+ user_compression_defined = true;
break;
case 0:
@@ -687,23 +690,20 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /*
+ * Custom and directory formats are compressed by default (zlib), others
+ * not
+ */
+ if (user_compression_defined == false)
{
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, &compress_spec);
#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ parse_compress_specification(PG_COMPRESSION_GZIP, NULL,
+ &compress_spec);
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -716,8 +716,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compress_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -948,10 +948,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compress_spec = compress_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -998,7 +995,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1258,6 +1256,61 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+/*
+ * Interprets and validates a compression option using the common compression
+ * parsing functions. If the requested compression is not available then the
+ * archives are uncompressed.
+ */
+static bool
+parse_compression(const char *opt, pg_compress_specification *compress_spec)
+{
+ char *algorithm_str = NULL;
+ char *level_str = NULL;
+ char *validation_error = NULL;
+ bool supports_compression = true;
+
+ parse_compress_options(opt, &algorithm_str, &level_str);
+ if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
+ {
+ pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ algorithm_str);
+ return false;
+ }
+
+ /* Switch off unimplemented or unavailable compressions. */
+ if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
+ compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ supports_compression = false;
+
+#ifndef HAVE_LIBZ
+ if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, compress_spec);
+
+ pg_free(algorithm_str);
+ pg_free(level_str);
+
+ return true;
+ }
+
+ /* Parse and validate the rest of the options */
+ parse_compress_specification(compress_spec->algorithm, level_str,
+ compress_spec);
+ validation_error = validate_compress_specification(compress_spec);
+ if (validation_error)
+ {
+ pg_log_error("invalid compression specification: %s", validation_error);
+ return false;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..f8d0b2fce5 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,16 +121,32 @@ command_fails_like(
'pg_restore: cannot specify both --single-transaction and multiple jobs');
command_fails_like(
- [ 'pg_dump', '-Z', '-1' ],
- qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
- 'pg_dump: -Z/--compress must be in range');
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level');
+
if (check_pg_config("#define HAVE_LIBZ 1"))
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '15' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
else
{
@@ -139,6 +155,11 @@ else
[ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: compression not available in this installation');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ],
+ qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index fe53ed0f89..d604558f03 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-01 02:05 ` Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-12-01 02:05 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Wed, Nov 30, 2022 at 05:11:44PM +0000, [email protected] wrote:
> Fair enough. The atteched v11 does that. 0001 introduces compression
> specification and is using it throughout. 0002 paves the way to the
> new interface by homogenizing the use of cfp. 0003 introduces the new
> API and stores the compression algorithm in the custom format header
> instead of the compression level integer. Finally 0004 adds support for
> LZ4.
I have been looking at 0001, and.. Hmm. I am really wondering
whether it would not be better to just nuke this warning into orbit.
This stuff enforces non-compression even if -Z has been used to a
non-default value. This has been moved to its current location by
cae2bb1 as of this thread:
https://www.postgresql.org/message-id/20160526.185551.242041780.horiguchi.kyotaro%40lab.ntt.co.jp
However, this is only active if -Z is used when not building with
zlib. At the end, it comes down to whether we want to prioritize the
portability of pg_dump commands specifying a -Z/--compress across
environments knowing that these may or may not be built with zlib,
vs the amount of simplification/uniformity we would get across the
binaries in the tree once we switch everything to use the compression
specifications. Now that pg_basebackup and pg_receivewal are managed
by compression specifications, and that we'd want more compression
options for pg_dump, I would tend to do the latter and from now on
complain if attempting to do a pg_dump -Z under --without-zlib with a
compression level > 0. zlib is also widely available, and we don't
document the fact that non-compression is enforced in this case,
either. (Two TAP tests with the custom format had to be tweaked.)
As per the patch, it is true that we do not need to bump the format of
the dump archives, as we can still store only the compression level
and guess the method from it. I have added some notes about that in
ReadHead and WriteHead to not forget.
Most of the changes are really-straight forward, and it has resisted
my tests, so I think that this is in a rather-commitable shape as-is.
--
Michael
Attachments:
[text/x-diff] v12-0001-Teach-pg_dump-about-compress_spec-and-use-it-thr.patch (37.5K, ../../[email protected]/2-v12-0001-Teach-pg_dump-about-compress_spec-and-use-it-thr.patch)
download | inline diff:
From a4fa522d0259e8969cde32798a917321cced0415 Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Thu, 1 Dec 2022 11:03:41 +0900
Subject: [PATCH v12] Teach pg_dump about compress_spec and use it throughout.
Align pg_dump with the rest of the binaries which use common compression. It is
teaching pg_dump.c about the common compression definitions and interfaces. Then
it propagates those throughout the code.
---
src/bin/pg_dump/compress_io.c | 107 ++++++++------------
src/bin/pg_dump/compress_io.h | 20 ++--
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 76 +++++++++-----
src/bin/pg_dump/pg_backup_archiver.h | 10 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 ++-
src/bin/pg_dump/pg_backup_tar.c | 11 +-
src/bin/pg_dump/pg_dump.c | 67 +++++++-----
src/bin/pg_dump/t/001_basic.pl | 34 +++++--
src/bin/pg_dump/t/002_pg_dump.pl | 3 +-
src/test/modules/test_pg_dump/t/001_base.pl | 16 +++
doc/src/sgml/ref/pg_dump.sgml | 34 +++++--
src/tools/pgindent/typedefs.list | 1 -
14 files changed, 242 insertions(+), 163 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..8f0d6d6210 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_specification compression_spec;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compression_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compression_spec = compression_spec;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, cs->compression_spec.level);
#endif
return cs;
@@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
+ if (compression_spec.algorithm == PG_COMPRESSION_NONE)
ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
#ifdef HAVE_LIBZ
ReadDataFromArchiveZlib(AH, readF);
@@ -179,18 +146,23 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compression_spec.algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -201,7 +173,7 @@ void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
+ if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
EndCompressorZlib(AH, cs);
#endif
free(cs);
@@ -453,20 +425,27 @@ cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compression_spec = {0};
+
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ {
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ fp = cfopen(path, mode, compression_spec);
+ }
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compression_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ fp = cfopen(fname, mode, compression_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +458,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compression_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compression_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -509,26 +489,27 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
+ * Opens file 'path' in 'mode'. If compression is GZIP, the file
* is opened with libz gzopen(), otherwise with plain fopen().
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression_spec.level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..6fad6c2cd5 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compression_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compression_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compression_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index e8b7898297..bc6b6594af 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compression_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index f39c0fa36f..22238539fd 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +99,8 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compression_spec);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -239,12 +241,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +257,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compression_spec = {0};
+
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compression_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -384,7 +392,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +468,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compression_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +748,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +978,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compression_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compression_spec.level = 0;
return opts;
}
@@ -1115,23 +1126,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compression_spec = {0};
teSection curSection;
OutputContext sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compression_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compression_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1485,7 +1501,8 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compression_spec)
{
int fn;
@@ -1508,12 +1525,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression_spec.level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2198,7 +2215,8 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2249,7 +2267,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression_spec = compression_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2264,7 +2282,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3669,7 +3687,12 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ /*
+ * For now the compression type is implied by the level. This will need
+ * to change once support for more compression algorithms is added,
+ * requiring a format bump.
+ */
+ WriteInt(AH, AH->compression_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3740,18 +3763,23 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ /* Guess the compression method based on the level */
+ AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression_spec.level = ReadInt(AH);
+
+ if (AH->compression_spec.level != 0)
+ AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 42687c4ec8..a9560c6045 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -331,14 +331,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compression_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index a0a55a1edd..f413d01fcb 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 798182b6f7..53ef8db728 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compression_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compression_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 402b93c610..99f3f5bcae 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -194,7 +194,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +328,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +383,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +401,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +800,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +887,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index da427f4d4a..510555ef21 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -105,6 +105,8 @@ static Oid g_last_builtin_oid; /* value of the last builtin oid */
/* The specified names/patterns should to match at least one entity */
static int strict_names = 0;
+static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE;
+
/*
* Object inclusion/exclusion lists
*
@@ -340,10 +342,13 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
+ pg_compress_specification compression_spec = {0};
+ char *compression_detail = NULL;
+ char *compression_algorithm_str = "none";
+ char *error_detail = NULL;
static DumpOptions dopt;
@@ -561,10 +566,9 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
- exit_nicely(1);
+ case 'Z': /* Compression */
+ parse_compress_options(optarg, &compression_algorithm_str,
+ &compression_detail);
break;
case 0:
@@ -687,22 +691,33 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
- {
-#ifdef HAVE_LIBZ
- if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
-#endif
- compressLevel = 0;
- }
+ /*
+ * Compression options
+ */
+ if (!parse_compress_algorithm(compression_algorithm_str,
+ &compression_algorithm))
+ pg_fatal("unrecognized compression algorithm: \"%s\"",
+ compression_algorithm_str);
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
+ parse_compress_specification(compression_algorithm, compression_detail,
+ &compression_spec);
+ error_detail = validate_compress_specification(&compression_spec);
+ if (error_detail != NULL)
+ pg_fatal("invalid compression specification: %s",
+ error_detail);
+
+ switch (compression_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ case PG_COMPRESSION_GZIP:
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ }
/*
* If emitting an archive format, we always want to emit a DATABASE item,
@@ -716,8 +731,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compression_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -948,10 +963,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression_spec = compression_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -998,7 +1010,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..c8bc02126d 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,24 +121,46 @@ command_fails_like(
'pg_restore: cannot specify both --single-transaction and multiple jobs');
command_fails_like(
- [ 'pg_dump', '-Z', '-1' ],
- qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
- 'pg_dump: -Z/--compress must be in range');
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: unrecognized compression algorithm/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level'
+);
+
if (check_pg_config("#define HAVE_LIBZ 1"))
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '15' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
else
{
# --jobs > 1 forces an error with tar format.
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
- qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
- 'pg_dump: warning: compression not available in this installation');
+ [ 'pg_dump', '--format', 'tar', '-j3' ],
+ qr/\Qpg_dump: error: parallel backup only supported by the directory format\E/,
+ 'pg_dump: warning: parallel backup not supported by tar format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index fe53ed0f89..709db0986d 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
@@ -200,6 +200,7 @@ my %pgdump_runs = (
# Do not use --no-sync to give test coverage for data sync.
defaults_custom_format => {
test_key => 'defaults',
+ compile_option => 'gzip',
dump_cmd => [
'pg_dump', '-Fc', '-Z6',
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl
index f5da6bf46d..19577ce0ea 100644
--- a/src/test/modules/test_pg_dump/t/001_base.pl
+++ b/src/test/modules/test_pg_dump/t/001_base.pl
@@ -20,6 +20,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# to define how each test should (or shouldn't) treat a result
# from a given run.
#
+# compile_option indicates if the commands run depend on a compilation
+# option, if any. This can be used to control if tests should be
+# skipped when a build dependency is not satisfied.
+#
# test_key indicates that a given run should simply use the same
# set of like/unlike tests as another run, and which run that is.
#
@@ -90,6 +94,7 @@ my %pgdump_runs = (
},
defaults_custom_format => {
test_key => 'defaults',
+ compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--no-sync', '-Fc', '-Z6',
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
@@ -749,6 +754,8 @@ $node->start;
my $port = $node->port;
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+
#########################################
# Set up schemas, tables, etc, to be dumped.
@@ -792,6 +799,15 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
+ # Skip command-level tests for gzip if there is no support for it.
+ if ( defined($pgdump_runs{$run}->{compile_option})
+ && $pgdump_runs{$run}->{compile_option} eq 'gzip'
+ && !$supports_gzip)
+ {
+ note "$run: skipped due to no gzip support";
+ next;
+ }
+
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8b9d9f4cad..2a015908f0 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,35 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>).
+ </para>
+ <para>
+ If no compression level is specified, the default compression
+ level will be used. If only a level is specified without mentioning
+ an algorithm, <literal>gzip</literal> compression will be used if
+ the level is greater than <literal>0</literal>, and no compression
+ will be used if the level is <literal>0</literal>.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 2f5802195d..58daeca831 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -428,7 +428,6 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
-CompressionAlgorithm
CompressionLocation
CompressorState
ComputeXidHorizonsResult
--
2.38.1
[application/pgp-signature] signature.asc (833B, ../../[email protected]/3-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-01 14:58 ` [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-01 14:58 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Thursday, December 1st, 2022 at 3:05 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Wed, Nov 30, 2022 at 05:11:44PM +0000, [email protected] wrote:
>
> > Fair enough. The atteched v11 does that. 0001 introduces compression
> > specification and is using it throughout. 0002 paves the way to the
> > new interface by homogenizing the use of cfp. 0003 introduces the new
> > API and stores the compression algorithm in the custom format header
> > instead of the compression level integer. Finally 0004 adds support for
> > LZ4.
>
>
> I have been looking at 0001, and.. Hmm. I am really wondering
> whether it would not be better to just nuke this warning into orbit.
> This stuff enforces non-compression even if -Z has been used to a
> non-default value. This has been moved to its current location by
> cae2bb1 as of this thread:
> https://www.postgresql.org/message-id/20160526.185551.242041780.horiguchi.kyotaro%40lab.ntt.co.jp
>
> However, this is only active if -Z is used when not building with
> zlib. At the end, it comes down to whether we want to prioritize the
> portability of pg_dump commands specifying a -Z/--compress across
> environments knowing that these may or may not be built with zlib,
> vs the amount of simplification/uniformity we would get across the
> binaries in the tree once we switch everything to use the compression
> specifications. Now that pg_basebackup and pg_receivewal are managed
> by compression specifications, and that we'd want more compression
> options for pg_dump, I would tend to do the latter and from now on
> complain if attempting to do a pg_dump -Z under --without-zlib with a
> compression level > 0. zlib is also widely available, and we don't
> document the fact that non-compression is enforced in this case,
> either. (Two TAP tests with the custom format had to be tweaked.)
Fair enough. Thank you for looking. However I have a small comment
on your new patch.
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
- {
-#ifdef HAVE_LIBZ
- if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
-#endif
- compressLevel = 0;
- }
Nuking the warning from orbit and changing the behaviour around disabling
the requested compression when the libraries are not present, should not
mean that we need to change the behaviour of default values for different
formats. Please find v13 attached which reinstates it.
Which in itself it got me looking and wondering why the tests succeeded.
The only existing test covering that path is `defaults_dir_format` in
`002_pg_dump.pl`. However as the test is currently written it does not
check whether the output was compressed. The restore command would succeed
in either case. A simple `gzip -t -r` against the directory will not
suffice to test it, because there exist files which are never compressed
in this format (.toc). A little bit more involved test case would need
to be written, yet before I embark to this journey, I would like to know
if you would agree to reinstate the defaults for those formats.
>
> As per the patch, it is true that we do not need to bump the format of
> the dump archives, as we can still store only the compression level
> and guess the method from it. I have added some notes about that in
> ReadHead and WriteHead to not forget.
Agreed. A minor suggestion if you may.
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
It would seem a more consistent to error out in this case. We do error
in all other cases where the compression is not available.
>
> Most of the changes are really-straight forward, and it has resisted
> my tests, so I think that this is in a rather-commitable shape as-is.
Thank you.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v13-0001-Teach-pg_dump-about-compress_spec-and-use-it-thr.patch (38.0K, ../../jBiqIGoBbxyqLL2syvEDLm1W_TPukmBtAOkFyICkAssd8qupR7zjpA066Cj_syxQOVFsQORr3fc-g_WeATSXvdoybB-e7Xj26Y-C8U7rTKA=@pm.me/2-v13-0001-Teach-pg_dump-about-compress_spec-and-use-it-thr.patch)
download | inline diff:
From 16e10b38cc8eb6eb5b1ffc15365d7e6ce23eef0a Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Thu, 1 Dec 2022 08:58:51 +0000
Subject: [PATCH v13] Teach pg_dump about compress_spec and use it throughout.
Align pg_dump with the rest of the binaries which use common compression. It is
teaching pg_dump.c about the common compression definitions and interfaces. Then
it propagates those throughout the code.
---
doc/src/sgml/ref/pg_dump.sgml | 34 +++++--
src/bin/pg_dump/compress_io.c | 107 ++++++++------------
src/bin/pg_dump/compress_io.h | 20 ++--
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 78 +++++++++-----
src/bin/pg_dump/pg_backup_archiver.h | 10 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 ++-
src/bin/pg_dump/pg_backup_tar.c | 11 +-
src/bin/pg_dump/pg_dump.c | 79 ++++++++++-----
src/bin/pg_dump/t/001_basic.pl | 34 +++++--
src/bin/pg_dump/t/002_pg_dump.pl | 3 +-
src/test/modules/test_pg_dump/t/001_base.pl | 16 +++
src/tools/pgindent/typedefs.list | 1 -
14 files changed, 258 insertions(+), 161 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8b9d9f4cad..2a015908f0 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,35 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>).
+ </para>
+ <para>
+ If no compression level is specified, the default compression
+ level will be used. If only a level is specified without mentioning
+ an algorithm, <literal>gzip</literal> compression will be used if
+ the level is greater than <literal>0</literal>, and no compression
+ will be used if the level is <literal>0</literal>.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..8f0d6d6210 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -64,7 +64,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_specification compression_spec;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compression_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compression_spec = compression_spec;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, cs->compression_spec.level);
#endif
return cs;
@@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
+ if (compression_spec.algorithm == PG_COMPRESSION_NONE)
ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
#ifdef HAVE_LIBZ
ReadDataFromArchiveZlib(AH, readF);
@@ -179,18 +146,23 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compression_spec.algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -201,7 +173,7 @@ void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
+ if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
EndCompressorZlib(AH, cs);
#endif
free(cs);
@@ -453,20 +425,27 @@ cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compression_spec = {0};
+
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ {
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ fp = cfopen(path, mode, compression_spec);
+ }
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compression_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ fp = cfopen(fname, mode, compression_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +458,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compression_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compression_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -509,26 +489,27 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
+ * Opens file 'path' in 'mode'. If compression is GZIP, the file
* is opened with libz gzopen(), otherwise with plain fopen().
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression_spec.level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..6fad6c2cd5 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compression_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compression_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compression_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index e8b7898297..bc6b6594af 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compression_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index f39c0fa36f..ac647662a3 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +99,8 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compression_spec);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -239,12 +241,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +257,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compression_spec = {0};
+
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compression_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -384,7 +392,8 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -459,8 +468,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compression_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +748,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +978,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compression_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compression_spec.level = 0;
return opts;
}
@@ -1115,23 +1126,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compression_spec = {0};
teSection curSection;
OutputContext sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compression_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compression_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1485,7 +1501,8 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compression_spec)
{
int fn;
@@ -1508,12 +1525,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression_spec.level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2198,7 +2215,8 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compression_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2249,7 +2267,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression_spec = compression_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2264,7 +2282,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3669,7 +3687,12 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ /*
+ * For now the compression type is implied by the level. This will need
+ * to change once support for more compression algorithms is added,
+ * requiring a format bump.
+ */
+ WriteInt(AH, AH->compression_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3740,19 +3763,24 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ /* Guess the compression method based on the level */
+ AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression_spec.level = ReadInt(AH);
+
+ if (AH->compression_spec.level != 0)
+ AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
- pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ pg_fatal("archive is compressed, but this installation does not support compression");
#endif
if (AH->version >= K_VERS_1_4)
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 42687c4ec8..a9560c6045 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -331,14 +331,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compression_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index a0a55a1edd..f413d01fcb 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 798182b6f7..53ef8db728 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compression_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compression_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 402b93c610..99f3f5bcae 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -194,7 +194,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +328,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +383,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +401,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +800,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +887,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index da427f4d4a..f31ed21afa 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -105,6 +105,8 @@ static Oid g_last_builtin_oid; /* value of the last builtin oid */
/* The specified names/patterns should to match at least one entity */
static int strict_names = 0;
+static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE;
+
/*
* Object inclusion/exclusion lists
*
@@ -340,10 +342,14 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
+ pg_compress_specification compression_spec = {0};
+ char *compression_detail = NULL;
+ char *compression_algorithm_str = "none";
+ char *error_detail = NULL;
+ bool user_compression_defined = false;
static DumpOptions dopt;
@@ -561,10 +567,10 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
- exit_nicely(1);
+ case 'Z': /* Compression */
+ parse_compress_options(optarg, &compression_algorithm_str,
+ &compression_detail);
+ user_compression_defined = true;
break;
case 0:
@@ -687,23 +693,50 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /*
+ * Compression options
+ */
+ if (!parse_compress_algorithm(compression_algorithm_str,
+ &compression_algorithm))
+ pg_fatal("unrecognized compression algorithm: \"%s\"",
+ compression_algorithm_str);
+
+ parse_compress_specification(compression_algorithm, compression_detail,
+ &compression_spec);
+ error_detail = validate_compress_specification(&compression_spec);
+ if (error_detail != NULL)
+ pg_fatal("invalid compression specification: %s",
+ error_detail);
+
+ switch (compression_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ /* fallthrough */
+ case PG_COMPRESSION_GZIP:
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ }
+
+ /*
+ * Custom and directory formats are compressed by default (zlib), others
+ * not
+ */
+ if ((archiveFormat == archCustom || archiveFormat == archDirectory) &&
+ !user_compression_defined)
{
#ifdef HAVE_LIBZ
- if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ parse_compress_specification(PG_COMPRESSION_GZIP, NULL,
+ &compression_spec);
+#else
+ /* no op */
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -716,8 +749,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compression_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -948,10 +981,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression_spec = compression_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -998,7 +1028,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..c8bc02126d 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,24 +121,46 @@ command_fails_like(
'pg_restore: cannot specify both --single-transaction and multiple jobs');
command_fails_like(
- [ 'pg_dump', '-Z', '-1' ],
- qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
- 'pg_dump: -Z/--compress must be in range');
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: unrecognized compression algorithm/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level'
+);
+
if (check_pg_config("#define HAVE_LIBZ 1"))
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '15' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
else
{
# --jobs > 1 forces an error with tar format.
command_fails_like(
- [ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
- qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
- 'pg_dump: warning: compression not available in this installation');
+ [ 'pg_dump', '--format', 'tar', '-j3' ],
+ qr/\Qpg_dump: error: parallel backup only supported by the directory format\E/,
+ 'pg_dump: warning: parallel backup not supported by tar format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index fe53ed0f89..709db0986d 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
@@ -200,6 +200,7 @@ my %pgdump_runs = (
# Do not use --no-sync to give test coverage for data sync.
defaults_custom_format => {
test_key => 'defaults',
+ compile_option => 'gzip',
dump_cmd => [
'pg_dump', '-Fc', '-Z6',
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
diff --git a/src/test/modules/test_pg_dump/t/001_base.pl b/src/test/modules/test_pg_dump/t/001_base.pl
index f5da6bf46d..19577ce0ea 100644
--- a/src/test/modules/test_pg_dump/t/001_base.pl
+++ b/src/test/modules/test_pg_dump/t/001_base.pl
@@ -20,6 +20,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# to define how each test should (or shouldn't) treat a result
# from a given run.
#
+# compile_option indicates if the commands run depend on a compilation
+# option, if any. This can be used to control if tests should be
+# skipped when a build dependency is not satisfied.
+#
# test_key indicates that a given run should simply use the same
# set of like/unlike tests as another run, and which run that is.
#
@@ -90,6 +94,7 @@ my %pgdump_runs = (
},
defaults_custom_format => {
test_key => 'defaults',
+ compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--no-sync', '-Fc', '-Z6',
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
@@ -749,6 +754,8 @@ $node->start;
my $port = $node->port;
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+
#########################################
# Set up schemas, tables, etc, to be dumped.
@@ -792,6 +799,15 @@ foreach my $run (sort keys %pgdump_runs)
my $test_key = $run;
+ # Skip command-level tests for gzip if there is no support for it.
+ if ( defined($pgdump_runs{$run}->{compile_option})
+ && $pgdump_runs{$run}->{compile_option} eq 'gzip'
+ && !$supports_gzip)
+ {
+ note "$run: skipped due to no gzip support";
+ next;
+ }
+
$node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
"$run: pg_dump runs");
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 2f5802195d..58daeca831 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -428,7 +428,6 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
-CompressionAlgorithm
CompressionLocation
CompressorState
ComputeXidHorizonsResult
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-02 01:56 ` Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-12-02 01:56 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Thu, Dec 01, 2022 at 02:58:35PM +0000, [email protected] wrote:
> Nuking the warning from orbit and changing the behaviour around disabling
> the requested compression when the libraries are not present, should not
> mean that we need to change the behaviour of default values for different
> formats. Please find v13 attached which reinstates it.
Gah, thanks! And this default behavior is documented as dependent on
the compilation as well.
> Which in itself it got me looking and wondering why the tests succeeded.
> The only existing test covering that path is `defaults_dir_format` in
> `002_pg_dump.pl`. However as the test is currently written it does not
> check whether the output was compressed. The restore command would succeed
> in either case. A simple `gzip -t -r` against the directory will not
> suffice to test it, because there exist files which are never compressed
> in this format (.toc). A little bit more involved test case would need
> to be written, yet before I embark to this journey, I would like to know
> if you would agree to reinstate the defaults for those formats.
On top of my mind, I briefly recall that -r is not that portable. And
the toc format makes the files generated non-deterministic as these
use OIDs..
[.. thinks ..]
We are going to need a new thing here, as compress_cmd cannot be
directly used. What if we used only an array of glob()-able elements?
Let's say "expected_contents" that could include a "dir_path/*.gz"
conditional on $supports_gzip? glob() can only be calculated when the
test is run as the file names cannot be known beforehand :/
>> As per the patch, it is true that we do not need to bump the format of
>> the dump archives, as we can still store only the compression level
>> and guess the method from it. I have added some notes about that in
>> ReadHead and WriteHead to not forget.
>
> Agreed. A minor suggestion if you may.
>
> #ifndef HAVE_LIBZ
> - if (AH->compression != 0)
> + if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
> pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
> #endif
>
> It would seem a more consistent to error out in this case. We do error
> in all other cases where the compression is not available.
Makes sense.
I have gone through the patch again, and applied it. Thanks!
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-02 16:15 ` [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-02 16:15 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Friday, December 2nd, 2022 at 2:56 AM, Michael Paquier <[email protected]> wrote:
> On top of my mind, I briefly recall that -r is not that portable. And
> the toc format makes the files generated non-deterministic as these
> use OIDs..
>
> [.. thinks ..]
>
> We are going to need a new thing here, as compress_cmd cannot be
> directly used. What if we used only an array of glob()-able elements?
> Let's say "expected_contents" that could include a "dir_path/*.gz"
> conditional on $supports_gzip? glob() can only be calculated when the
> test is run as the file names cannot be known beforehand :/
You are very correct. However one can glob after the fact. Please find
0001 of the attached v14 which attempts to implement it.
> I have gone through the patch again, and applied it. Thanks!
Thank you. Please find the rest of of the patchset series rebased on top
of it. I dare to say that 0002 is in a state worth of your consideration.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v14-0001-Provide-coverage-for-pg_dump-default-compression.patch (1.7K, ../../T33dciWfZ3WaU10Mspm2_vsftrl0DI_cs4rQcrT_pLubaLq_HCRlW5QhEOT1cWtSRtdeTVXrL2r2BTYz3Fjz_3YNh6i77lwd5AxNxb5oanQ=@pm.me/2-v14-0001-Provide-coverage-for-pg_dump-default-compression.patch)
download | inline diff:
From fdab9843ba84d64e96e461c9f8e78a932cc366e1 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:02:51 +0000
Subject: [PATCH v14 1/4] Provide coverage for pg_dump default compression for
dir format
The restore program will succeed regardless of whether the dumped output was
compressed or not. This commit implements a portable way to check the contents
of the directory via perl's build in filename expansion.
---
src/bin/pg_dump/t/002_pg_dump.pl | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 709db0986d..03b5375e70 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -215,6 +215,7 @@ my %pgdump_runs = (
# Do not use --no-sync to give test coverage for data sync.
defaults_dir_format => {
test_key => 'defaults',
+ compile_option => 'gzip',
dump_cmd => [
'pg_dump', '-Fd',
"--file=$tempdir/defaults_dir_format", 'postgres',
@@ -224,6 +225,7 @@ my %pgdump_runs = (
"--file=$tempdir/defaults_dir_format.sql",
"$tempdir/defaults_dir_format",
],
+ glob_pattern => "$tempdir/defaults_dir_format/*.dat.gz"
},
# Do not use --no-sync to give test coverage for data sync.
@@ -4153,6 +4155,13 @@ foreach my $run (sort keys %pgdump_runs)
command_ok(\@full_compress_cmd, "$run: compression commands");
}
+ if ($pgdump_runs{$run}->{glob_pattern})
+ {
+ my $glob_pattern = $pgdump_runs{$run}->{glob_pattern};
+ my @glob_output = glob($glob_pattern);
+ is(scalar(@glob_output) > 0, 1, "glob pattern matched")
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.34.1
[text/x-patch] v14-0002-Prepare-pg_dump-internals-for-additional-compres.patch (19.9K, ../../T33dciWfZ3WaU10Mspm2_vsftrl0DI_cs4rQcrT_pLubaLq_HCRlW5QhEOT1cWtSRtdeTVXrL2r2BTYz3Fjz_3YNh6i77lwd5AxNxb5oanQ=@pm.me/3-v14-0002-Prepare-pg_dump-internals-for-additional-compres.patch)
download | inline diff:
From b87e02c847c84fdd8823b188fd5cf1b28e5b9099 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:02:58 +0000
Subject: [PATCH v14 2/4] Prepare pg_dump internals for additional compression
methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about it and is using it throughout.
---
src/bin/pg_dump/compress_io.c | 363 ++++++++++++++++++---------
src/bin/pg_dump/pg_backup_archiver.c | 128 ++++------
src/bin/pg_dump/pg_backup_archiver.h | 27 +-
3 files changed, 296 insertions(+), 222 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 8f0d6d6210..e453443b6a 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -127,15 +131,23 @@ void
ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec,
ReadFunc readF)
{
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ switch (compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -172,11 +184,24 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -390,10 +415,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_specification compression_spec;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -489,127 +512,195 @@ cfopen_write(const char *path, const char *mode,
}
/*
- * Opens file 'path' in 'mode'. If compression is GZIP, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ fp->compression_spec = compression_spec;
+
+ switch (compression_spec.algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compression_spec.level);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(path, -1, mode, compression_spec);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(NULL, fd, mode, compression_spec);
+}
int
cfread(void *ptr, int size, cfp *fp)
{
- int ret;
+ int ret = 0;
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfgetc(cfp *fp)
{
- int ret;
+ int ret = 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -618,65 +709,113 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret = NULL;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret = 0;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compression_spec.algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 0081873a72..e1652ad013 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -101,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -277,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -362,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -391,17 +383,24 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -1128,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
@@ -1503,58 +1502,32 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression_spec.level);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compression_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compression_spec);
if (!AH->OF)
{
@@ -1565,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename,
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1715,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2219,6 +2178,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2272,8 +2232,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a9560c6045..ad65693242 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
--
2.34.1
[text/x-patch] v14-0003-Introduce-Compressor-API-in-pg_dump.patch (56.4K, ../../T33dciWfZ3WaU10Mspm2_vsftrl0DI_cs4rQcrT_pLubaLq_HCRlW5QhEOT1cWtSRtdeTVXrL2r2BTYz3Fjz_3YNh6i77lwd5AxNxb5oanQ=@pm.me/4-v14-0003-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From a8f4dde046c1c955c8c386e554869bb667ddd600 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:03:05 +0000
Subject: [PATCH v14 3/4] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives now need to store the compression algorithm in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 390 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 9 +
src/bin/pg_dump/compress_io.c | 866 +++++++-------------------
src/bin/pg_dump/compress_io.h | 68 +-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 102 +--
src/bin/pg_dump/pg_backup_archiver.h | 4 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 85 +--
10 files changed, 783 insertions(+), 766 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..bc6d1abc77
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,390 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ int compressionLevel;
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, gzipcs->compressionLevel) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+typedef struct GzipData
+{
+ gzFile fp;
+ int compressionLevel;
+} GzipData;
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ size_t ret;
+
+ ret = gzread(gd->fp, ptr, size);
+ if (ret != size && !gzeof(gd->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gd->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzwrite(gd->fp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gd->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gd->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzgets(gd->fp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gd->fp);
+
+ save_errno = errno;
+ free(gd);
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzeof(gd->fp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gd->fp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ char mode_compression[32];
+
+ if (gd->compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, gd->compressionLevel);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gd->fp = gzdopen(dup(fd), mode_compression);
+ else
+ gd->fp = gzopen(path, mode_compression);
+
+ if (gd->fp == NULL)
+ return 1;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ GzipData *gd;
+
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ gd = pg_malloc0(sizeof(GzipData));
+ gd->compressionLevel = compressionLevel;
+
+ CFH->private = gd;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..ab0362c1f3
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, int compressionLevel);
+extern void InitCompressGzip(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index e453443b6a..96132116f9 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,700 +68,253 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+ static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_specification compression_spec;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
-
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
-
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
-/* Public interface routines */
-
-/* Allocate a new compressor */
-CompressorState *
-AllocateCompressor(const pg_compress_specification compression_spec,
- WriteFunc writeF)
-{
- CompressorState *cs;
-
-#ifndef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
-#endif
-
- cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
- cs->writeF = writeF;
- cs->compression_spec = compression_spec;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, cs->compression_spec.level);
-#endif
-
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec,
- ReadFunc readF)
-{
- switch (compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-}
-
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-}
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
-/*
- * Terminate compression library context and flush its buffers.
- */
-void
-EndCompressor(ArchiveHandle *AH, CompressorState *cs)
-{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- free(cs);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ free(buf);
}
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
- */
-
-static void
-InitCompressorZlib(CompressorState *cs, int level)
+ static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ cs->writeF(AH, data, dLen);
}
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
+ static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
+ /* no op */
}
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
+ static void
+InitCompressorNone(CompressorState *cs)
{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
-
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
-
- if (res == Z_STREAM_END)
- break;
- }
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
}
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
+/* Public interface routines */
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
+/* Allocate a new compressor */
+ CompressorState *
+AllocateCompressor(const pg_compress_specification compression_spec,
+ ReadFunc readF, WriteFunc writeF)
{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
+ CompressorState *cs;
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
+ cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
+ cs->writeF = writeF;
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
+ switch (compression_spec.algorithm)
{
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ case PG_COMPRESSION_NONE:
+ InitCompressorNone(cs);
+ break;
+ case PG_COMPRESSION_GZIP:
+ InitCompressorGzip(cs, compression_spec.level);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
- free(buf);
- free(out);
- free(zp);
+ return cs;
}
-#endif /* HAVE_LIBZ */
-
/*
- * Functions for uncompressed output.
+ * Terminate compression library context and flush its buffers.
*/
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
-{
- size_t cnt;
- char *buf;
- size_t buflen;
-
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
-
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
-
- free(buf);
-}
-
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+ void
+EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- cs->writeF(AH, data, dLen);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-
/*----------------------
* Compressed stream API
*----------------------
*/
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+ static int
+hasSuffix(const char *filename, const char *suffix)
{
- pg_compress_specification compression_spec;
- void *fp;
-};
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
+}
/* free() without changing errno; useful in several places below */
-static void
+ static void
free_keep_errno(void *p)
{
- int save_errno = errno;
+ int save_errno = errno;
- free(p);
- errno = save_errno;
+ free(p);
+ errno = save_errno;
}
/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
+ * Compression None implementation
*/
-cfp *
-cfopen_read(const char *path, const char *mode)
+ static size_t
+_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp;
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- pg_compress_specification compression_spec = {0};
+ if (size == 0)
+ return 0;
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- {
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(path, mode, compression_spec);
- }
- else
-#endif
- {
- compression_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compression_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
- fname = psprintf("%s.gz", path);
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
+ return ret;
}
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+ static size_t
+_write(const void *ptr, size_t size, CompressFileHandle * CFH)
{
- cfp *fp;
-
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compression_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_specification compression_spec)
+ static const char *
+_get_error(CompressFileHandle * CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
-
- fp->compression_spec = compression_spec;
-
- switch (compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return fp;
+ return strerror(errno);
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+ static char *
+_gets(char *ptr, int size, CompressFileHandle * CFH)
{
- return cfopen_internal(path, -1, mode, compression_spec);
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compression_spec)
+ static int
+_getc(CompressFileHandle * CFH)
{
- return cfopen_internal(NULL, fd, mode, compression_spec);
+ FILE *fp = (FILE *) CFH->private;
+ int ret;
+
+ ret = fgetc(fp);
+ if (ret == EOF)
+ {
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
}
-int
-cfread(void *ptr, int size, cfp *fp)
+ static int
+_close(CompressFileHandle * CFH)
{
- int ret = 0;
-
- if (size == 0)
- return 0;
+ FILE *fp = (FILE *) CFH->private;
+ int ret = 0;
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
+ CFH->private = NULL;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (fp)
+ ret = fclose(fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return ret;
+ return ret;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+ static int
+_eof(CompressFileHandle * CFH)
{
- int ret = 0;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return ret;
+ return feof((FILE *) CFH->private);
}
-int
-cfgetc(cfp *fp)
+ static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
{
- int ret = 0;
+ Assert(CFH->private == NULL);
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ if (CFH->private == NULL)
+ return 1;
- return ret;
+ return 0;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+ static int
+_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
{
- char *ret = NULL;
+ Assert(CFH->private == NULL);
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ return 0;
+}
- return ret;
+ static void
+InitCompressNone(CompressFileHandle * CFH)
+{
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-int
-cfclose(cfp *fp)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compression_spec)
{
- int ret = 0;
+ CompressFileHandle *CFH;
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
- switch (fp->compression_spec.algorithm)
+ switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
+ InitCompressNone(CFH);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressGzip(CFH, compression_spec.level);
break;
case PG_COMPRESSION_LZ4:
/* fallthrough */
@@ -767,71 +323,77 @@ cfclose(cfp *fp)
break;
}
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
-int
-cfeof(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret = 0;
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compression_spec = {0};
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- return ret;
-}
+ fname = strdup(path);
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ if (hasSuffix(fname, ".gz"))
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
+ bool exists;
+
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("not built with zlib support");
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
}
- return strerror(errno);
+ CFH = InitCompressFileHandle(compression_spec);
+ if (CFH->open(fname, -1, mode, CFH))
+ {
+ free_keep_errno(CFH);
+ CFH = NULL;
+ }
+ free_keep_errno(fname);
+
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle * CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index 6fad6c2cd5..1118b7a638 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,60 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle * cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle * CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle * CFH);
+ int (*getc) (CompressFileHandle * CFH);
+ int (*eof) (CompressFileHandle * CFH);
+ int (*close) (CompressFileHandle * CFH);
+ const char *(*get_error) (CompressFileHandle * CFH);
+
+ void *private;
+};
-typedef struct cfp cfp;
+extern CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compress_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compression_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle * InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle * CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index d96e566846..0c73a4707e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index e1652ad013..989f276301 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle * SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1127,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1143,9 +1143,10 @@ PrintTOCSummary(Archive *AHX)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression_spec.level);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1502,6 +1503,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1526,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compression_spec);
- else
- AH->OF = cfopen(filename, mode, compression_spec);
+ CFH = InitCompressFileHandle(compression_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1690,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2036,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2078,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2181,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2237,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,12 +3653,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- /*
- * For now the compression type is implied by the level. This will need
- * to change once support for more compression algorithms is added,
- * requiring a format bump.
- */
- WriteInt(AH, AH->compression_spec.level);
+ AH->WriteBytePtr(AH, AH->compression_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3722,10 +3724,11 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
- /* Guess the compression method based on the level */
- AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compression_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
+ /* Guess the compression method based on the level */
if (AH->version < K_VERS_1_4)
AH->compression_spec.level = AH->ReadBytePtr(AH);
else
@@ -3737,10 +3740,17 @@ ReadHead(ArchiveHandle *AH)
else
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("archive is compressed, but this installation does not support compression");
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
+ if (unsupported)
+ pg_fatal("archive is compressed, but this installation does not support compression");
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index ad65693242..17591af7bc 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,12 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index f413d01fcb..3b461b048a 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compression_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 53ef8db728..b03127e720 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,9 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
+ CompressFileHandle *dataFH; /* currently open data file */
- cfp *blobsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *blobsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +198,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +218,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +327,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compression_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +346,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +371,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +386,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +435,7 @@ _LoadBlobs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +443,14 @@ _LoadBlobs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->blobsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the blobs TOC file line-by-line, and process each blob */
- while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char blobfname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +465,11 @@ _LoadBlobs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreBlob(AH, oid);
}
- if (!cfeof(ctx->blobsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +489,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +514,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +527,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +549,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +578,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +589,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compression_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +603,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +654,8 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
/* The blob TOC file is never compressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->blobsTocFH = cfopen_write(fname, "ab", compression_spec);
- if (ctx->blobsTocFH == NULL)
+ ctx->blobsTocFH = InitCompressFileHandle(compression_spec);
+ if (ctx->blobsTocFH->open_write(fname, "ab", ctx->blobsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +672,8 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,17 +686,18 @@ static void
_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->blobsTocFH;
char buf[50];
int len;
/* Close the BLOB data file itself */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the blob in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->blobsTocFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
pg_fatal("could not write to blobs TOC file");
}
@@ -706,7 +711,7 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->blobsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->blobsTocFH) != 0)
pg_fatal("could not close blobs TOC file: %m");
ctx->blobsTocFH = NULL;
}
--
2.34.1
[text/x-patch] v14-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (28.7K, ../../T33dciWfZ3WaU10Mspm2_vsftrl0DI_cs4rQcrT_pLubaLq_HCRlW5QhEOT1cWtSRtdeTVXrL2r2BTYz3Fjz_3YNh6i77lwd5AxNxb5oanQ=@pm.me/5-v14-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 0d16e54023485dcc36ee1252132c444fb4c6aa7f Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:03:12 +0000
Subject: [PATCH v14 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 13 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 79 ++--
src/bin/pg_dump/compress_lz4.c | 601 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 9 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 5 +-
src/bin/pg_dump/t/002_pg_dump.pl | 69 ++-
9 files changed, 753 insertions(+), 47 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 363d1327e2..ebf3e968d0 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -328,9 +328,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -652,7 +653,7 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression.
+ <literal>lz4</literal> or <literal>none</literal> for no compression.
A compression detail string can optionally be specified. If the
detail string is an integer, it specifies the compression level.
Otherwise, it should be a comma-separated list of items, each of the
@@ -673,8 +674,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 96132116f9..2bf77c693d 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -115,28 +118,29 @@ InitCompressorNone(CompressorState *cs)
AllocateCompressor(const pg_compress_specification compression_spec,
ReadFunc readF, WriteFunc writeF)
{
- CompressorState *cs;
+ CompressorState *cs;
- cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
- cs->readF = readF;
- cs->writeF = writeF;
+ cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
+ cs->writeF = writeF;
- switch (compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- InitCompressorNone(cs);
- break;
- case PG_COMPRESSION_GZIP:
- InitCompressorGzip(cs, compression_spec.level);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ switch (compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ InitCompressorNone(cs);
+ break;
+ case PG_COMPRESSION_GZIP:
+ InitCompressorGzip(cs, compression_spec.level);
+ break;
+ case PG_COMPRESSION_LZ4:
+ InitCompressorLZ4(cs, compression_spec.level);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
- return cs;
+ return cs;
}
/*
@@ -181,7 +185,8 @@ free_keep_errno(void *p)
/*
* Compression None implementation
*/
- static size_t
+
+static size_t
_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
FILE *fp = (FILE *) CFH->private;
@@ -317,7 +322,8 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
InitCompressGzip(CFH, compression_spec.level);
break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ InitCompressLZ4(CFH, compression_spec.level);
+ break;
case PG_COMPRESSION_ZSTD:
pg_fatal("invalid compression method");
break;
@@ -330,12 +336,12 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -371,6 +377,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compression_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..8f93f05e87
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,601 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*
+ * LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
+ * Redefine it for installations with a lesser version.
+ */
+#ifndef LZ4F_HEADER_SIZE_MAX
+#define LZ4F_HEADER_SIZE_MAX 32
+#endif
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..fbec9a508d
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, int compressionLevel);
+extern void InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 0c73a4707e..b27e92ffd0 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -83,7 +84,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 989f276301..6cf745bcc1 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -395,6 +395,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2074,7 +2078,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2084,6 +2088,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3747,6 +3755,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
pg_fatal("archive is compressed, but this installation does not support compression");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 44e8cd4704..8cb20c1bc0 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -713,13 +713,12 @@ main(int argc, char **argv)
case PG_COMPRESSION_NONE:
/* fallthrough */
case PG_COMPRESSION_GZIP:
+ /* fallthrough */
+ case PG_COMPRESSION_LZ4:
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
}
/*
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 03b5375e70..aedb6c994b 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -116,6 +116,67 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=1', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4130,11 +4191,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-03 02:45 ` Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-12-03 02:45 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Fri, Dec 02, 2022 at 04:15:10PM +0000, [email protected] wrote:
> You are very correct. However one can glob after the fact. Please find
> 0001 of the attached v14 which attempts to implement it.
+ if ($pgdump_runs{$run}->{glob_pattern})
+ {
+ my $glob_pattern = $pgdump_runs{$run}->{glob_pattern};
+ my @glob_output = glob($glob_pattern);
+ is(scalar(@glob_output) > 0, 1, "glob pattern matched")
+ }
While this is correct in checking that the contents are compressed
under --with-zlib, this also removes the coverage where we make sure
that this command is able to complete under --without-zlib without
compressing any of the table data files. Hence my point from
upthread: this test had better not use compile_option, but change
glob_pattern depending on if the build uses zlib or not.
In order to check this behavior with defaults_custom_format, perhaps
we could just remove the -Z6 from it or add an extra command for its
default behavior?
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-05 07:05 ` Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-12-05 07:05 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Sat, Dec 03, 2022 at 11:45:30AM +0900, Michael Paquier wrote:
> While this is correct in checking that the contents are compressed
> under --with-zlib, this also removes the coverage where we make sure
> that this command is able to complete under --without-zlib without
> compressing any of the table data files. Hence my point from
> upthread: this test had better not use compile_option, but change
> glob_pattern depending on if the build uses zlib or not.
In short, I mean something like the attached. I have named the flag
content_patterns, and switched it to an array so as we can check that
toc.dat is always uncompression and that the other data files are
always uncompressed.
> In order to check this behavior with defaults_custom_format, perhaps
> we could just remove the -Z6 from it or add an extra command for its
> default behavior?
This is slightly more complicated as there is just one file generated
for the compression and non-compression cases, so I have let that as
it is now.
--
Michael
Attachments:
[text/x-diff] v15-0001-Provide-coverage-for-pg_dump-default-compression.patch (3.4K, ../../Y42YMz%[email protected]/2-v15-0001-Provide-coverage-for-pg_dump-default-compression.patch)
download | inline diff:
From 5c583358caed5598fec9abea6750ff7fbd98d269 Mon Sep 17 00:00:00 2001
From: Michael Paquier <[email protected]>
Date: Mon, 5 Dec 2022 16:04:57 +0900
Subject: [PATCH v15] Provide coverage for pg_dump default compression for dir
format
The restore program will succeed regardless of whether the dumped output was
compressed or not. This commit implements a portable way to check the contents
of the directory via perl's build in filename expansion.
---
src/bin/pg_dump/t/002_pg_dump.pl | 29 +++++++++++++++++++++++++----
1 file changed, 25 insertions(+), 4 deletions(-)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 709db0986d..9796d2667f 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -36,6 +36,9 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# to test pg_restore's ability to parse manually compressed files
# that otherwise pg_dump does not compress on its own (e.g. *.toc).
#
+# content_patterns is an optional array consisting of strings compilable
+# with glob() to check the files generated after a dump.
+#
# restore_cmd is the pg_restore command to run, if any. Note
# that this should generally be used when the pg_dump goes to
# a non-text file and that the restore can then be used to
@@ -46,6 +49,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# database and then pg_dump *that* database (or something along
# those lines) to validate that part of the process.
+my $supports_icu = ($ENV{with_icu} eq 'yes');
+my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+
my %pgdump_runs = (
binary_upgrade => {
dump_cmd => [
@@ -213,6 +220,9 @@ my %pgdump_runs = (
},
# Do not use --no-sync to give test coverage for data sync.
+ # By default, the directory format compresses its contents
+ # when the code is compiled with gzip support, and lets things
+ # uncompressed when not compiled with it.
defaults_dir_format => {
test_key => 'defaults',
dump_cmd => [
@@ -224,6 +234,11 @@ my %pgdump_runs = (
"--file=$tempdir/defaults_dir_format.sql",
"$tempdir/defaults_dir_format",
],
+ content_patterns => ["$tempdir/defaults_dir_format/toc.dat",
+ $supports_gzip ?
+ "$tempdir/defaults_dir_format/*.dat.gz" :
+ "$tempdir/defaults_dir_format/*.dat",
+ ],
},
# Do not use --no-sync to give test coverage for data sync.
@@ -3920,10 +3935,6 @@ if ($collation_check_stderr !~ /ERROR: /)
$collation_support = 1;
}
-my $supports_icu = ($ENV{with_icu} eq 'yes');
-my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
-my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
-
# ICU doesn't work with some encodings
my $encoding = $node->safe_psql('postgres', 'show server_encoding');
$supports_icu = 0 if $encoding eq 'SQL_ASCII';
@@ -4153,6 +4164,16 @@ foreach my $run (sort keys %pgdump_runs)
command_ok(\@full_compress_cmd, "$run: compression commands");
}
+ if ($pgdump_runs{$run}->{content_patterns})
+ {
+ my $content_patterns = $pgdump_runs{$run}->{content_patterns};
+ foreach my $content_pattern (@{$content_patterns})
+ {
+ my @glob_output = glob($content_pattern);
+ is(scalar(@glob_output) > 0, 1, "$run: content check for $content_pattern");
+ }
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.38.1
[application/pgp-signature] signature.asc (833B, ../../Y42YMz%[email protected]/3-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-05 12:48 ` [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-05 12:48 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Monday, December 5th, 2022 at 8:05 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Sat, Dec 03, 2022 at 11:45:30AM +0900, Michael Paquier wrote:
>
> > While this is correct in checking that the contents are compressed
> > under --with-zlib, this also removes the coverage where we make sure
> > that this command is able to complete under --without-zlib without
> > compressing any of the table data files. Hence my point from
> > upthread: this test had better not use compile_option, but change
> > glob_pattern depending on if the build uses zlib or not.
>
> In short, I mean something like the attached. I have named the flag
> content_patterns, and switched it to an array so as we can check that
> toc.dat is always uncompression and that the other data files are
> always uncompressed.
I see. This approach is much better than my proposal, thanks. If you
allow me, I find 'content_patterns' to be slightly ambiguous. While is
true that it refers to the contents of a directory, it is not the
contents of the dump that it is examining. I took the liberty of proposing
an alternative name in the attached v16.
I also took the liberty of applying the test pattern when it the dump
is explicitly compressed.
> > In order to check this behavior with defaults_custom_format, perhaps
> > we could just remove the -Z6 from it or add an extra command for its
> > default behavior?
>
> This is slightly more complicated as there is just one file generated
> for the compression and non-compression cases, so I have let that as
> it is now.
I was thinking a bit more about this. I think that we can use the list
TOC option of pg_restore. This option will first print out the header
info which contains the compression. Perl utils already support to
parse the generated output of a command. Please find an attempt to do
so in the attached. The benefits of having some testing for this case
become a bit more obvious in 0004 of the patchset, when lz4 is
introduced.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v16-0001-Provide-coverage-for-pg_dump-default-compression.patch (5.3K, ../../DQn4czCWR1rcbGPLL7p3LfEr5-kGmlySm-H05VgroINdikvhtS5r9EdI6b8D8sjnbKdJ09k-cxs2AqijBeHAWk9Q8gvEAxPRHuLRhwONcGc=@pm.me/2-v16-0001-Provide-coverage-for-pg_dump-default-compression.patch)
download | inline diff:
From 75619245b02c7cd659d826d6ea8b3964445155a5 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 5 Dec 2022 12:14:22 +0000
Subject: [PATCH v16 1/4] Provide coverage for pg_dump default compression for
dir and custom format
The restore program will succeed regardless of whether the dumped output was
compressed or not. This commit implements a portable way to check the contents
of the directory via perl's build in filename expansion. It also implements a
way to check the custom's format data segmenets commpression by examining the
header information during the TOC summary output.
---
src/bin/pg_dump/t/002_pg_dump.pl | 65 +++++++++++++++++++++++++++++---
1 file changed, 59 insertions(+), 6 deletions(-)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index c2da1df39d..248540db8c 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -36,6 +36,9 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# to test pg_restore's ability to parse manually compressed files
# that otherwise pg_dump does not compress on its own (e.g. *.toc).
#
+# glob_patterns is an optional array consisting of strings compilable
+# with glob() to check the files generated after a dump.
+#
# restore_cmd is the pg_restore command to run, if any. Note
# that this should generally be used when the pg_dump goes to
# a non-text file and that the restore can then be used to
@@ -46,6 +49,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
# database and then pg_dump *that* database (or something along
# those lines) to validate that part of the process.
+my $supports_icu = ($ENV{with_icu} eq 'yes');
+my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+
my %pgdump_runs = (
binary_upgrade => {
dump_cmd => [
@@ -79,6 +86,14 @@ my %pgdump_runs = (
"--file=$tempdir/compression_gzip_custom.sql",
"$tempdir/compression_gzip_custom.dump",
],
+ command_like => {
+ command => [
+ 'pg_restore',
+ '-l', "$tempdir/compression_gzip_custom.dump",
+ ],
+ expected => qr/Compression: 1/,
+ name => 'data content is gzip compressed'
+ },
},
# Do not use --no-sync to give test coverage for data sync.
@@ -96,6 +111,11 @@ my %pgdump_runs = (
program => $ENV{'GZIP_PROGRAM'},
args => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
},
+ # Verify that only data files where compressed
+ glob_patterns => [
+ "$tempdir/compression_gzip_dir/toc.dat",
+ "$tempdir/compression_gzip_dir/*.dat.gz",
+ ],
restore_cmd => [
'pg_restore', '--jobs=2',
"--file=$tempdir/compression_gzip_dir.sql",
@@ -200,9 +220,8 @@ my %pgdump_runs = (
# Do not use --no-sync to give test coverage for data sync.
defaults_custom_format => {
test_key => 'defaults',
- compile_option => 'gzip',
dump_cmd => [
- 'pg_dump', '-Fc', '-Z6',
+ 'pg_dump', '-Fc',
"--file=$tempdir/defaults_custom_format.dump", 'postgres',
],
restore_cmd => [
@@ -210,9 +229,22 @@ my %pgdump_runs = (
"--file=$tempdir/defaults_custom_format.sql",
"$tempdir/defaults_custom_format.dump",
],
+ command_like => {
+ command => [
+ 'pg_restore',
+ '-l', "$tempdir/defaults_custom_format.dump",
+ ],
+ expected => $supports_gzip ?
+ qr/Compression: -1/ :
+ qr/Compression: 0/,
+ name => 'data content is gzip compressed by default if available'
+ },
},
# Do not use --no-sync to give test coverage for data sync.
+ # By default, the directory format compresses its data files
+ # when the code is compiled with gzip support, and lets them
+ # uncompressed when not compiled with it.
defaults_dir_format => {
test_key => 'defaults',
dump_cmd => [
@@ -224,6 +256,13 @@ my %pgdump_runs = (
"--file=$tempdir/defaults_dir_format.sql",
"$tempdir/defaults_dir_format",
],
+ glob_patterns => [
+ "$tempdir/defaults_dir_format/toc.dat",
+ "$tempdir/defaults_dir_format/blobs.toc",
+ $supports_gzip ?
+ "$tempdir/defaults_dir_format/*.dat.gz" :
+ "$tempdir/defaults_dir_format/*.dat",
+ ],
},
# Do not use --no-sync to give test coverage for data sync.
@@ -3920,10 +3959,6 @@ if ($collation_check_stderr !~ /ERROR: /)
$collation_support = 1;
}
-my $supports_icu = ($ENV{with_icu} eq 'yes');
-my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
-my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
-
# ICU doesn't work with some encodings
my $encoding = $node->safe_psql('postgres', 'show server_encoding');
$supports_icu = 0 if $encoding eq 'SQL_ASCII';
@@ -4153,6 +4188,24 @@ foreach my $run (sort keys %pgdump_runs)
command_ok(\@full_compress_cmd, "$run: compression commands");
}
+ if ($pgdump_runs{$run}->{glob_patterns})
+ {
+ my $glob_patterns = $pgdump_runs{$run}->{glob_patterns};
+ foreach my $glob_pattern (@{$glob_patterns})
+ {
+ my @glob_output = glob($glob_pattern);
+ is(scalar(@glob_output) > 0, 1, "$run: glob check for $glob_pattern");
+ }
+ }
+
+ if ($pgdump_runs{$run}->{command_like})
+ {
+ my $cmd_like = $pgdump_runs{$run}->{command_like};
+ $node->command_like(\@{ $cmd_like->{command} },
+ $cmd_like->{expected},
+ $cmd_like->{name})
+ }
+
if ($pgdump_runs{$run}->{restore_cmd})
{
$node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },
--
2.34.1
[text/x-patch] v16-0002-Prepare-pg_dump-internals-for-additional-compres.patch (19.9K, ../../DQn4czCWR1rcbGPLL7p3LfEr5-kGmlySm-H05VgroINdikvhtS5r9EdI6b8D8sjnbKdJ09k-cxs2AqijBeHAWk9Q8gvEAxPRHuLRhwONcGc=@pm.me/3-v16-0002-Prepare-pg_dump-internals-for-additional-compres.patch)
download | inline diff:
From c220ebbc01bf3694dbfb6dff94e3d2bff9b1bbc1 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:02:58 +0000
Subject: [PATCH v16 2/4] Prepare pg_dump internals for additional compression
methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about it and is using it throughout.
---
src/bin/pg_dump/compress_io.c | 363 ++++++++++++++++++---------
src/bin/pg_dump/pg_backup_archiver.c | 128 ++++------
src/bin/pg_dump/pg_backup_archiver.h | 27 +-
3 files changed, 296 insertions(+), 222 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index a7df600cc0..cb59300cb5 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -128,15 +132,23 @@ ReadDataFromArchive(ArchiveHandle *AH,
const pg_compress_specification compression_spec,
ReadFunc readF)
{
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ switch (compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -173,11 +185,24 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -391,10 +416,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_specification compression_spec;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -490,127 +513,195 @@ cfopen_write(const char *path, const char *mode,
}
/*
- * Opens file 'path' in 'mode'. If compression is GZIP, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ fp->compression_spec = compression_spec;
+
+ switch (compression_spec.algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compression_spec.level);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(path, -1, mode, compression_spec);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(NULL, fd, mode, compression_spec);
+}
int
cfread(void *ptr, int size, cfp *fp)
{
- int ret;
+ int ret = 0;
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfgetc(cfp *fp)
{
- int ret;
+ int ret = 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -619,65 +710,113 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret = NULL;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret = 0;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compression_spec.algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ /* fallthrough */
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7f7a0f1ce7..fb94317ad9 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -101,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -277,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -362,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -391,17 +383,24 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -1128,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
@@ -1503,58 +1502,32 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression_spec.level);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compression_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compression_spec);
if (!AH->OF)
{
@@ -1565,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename,
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1715,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2219,6 +2178,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2272,8 +2232,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index f72446ed5b..4725e49747 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
--
2.34.1
[text/x-patch] v16-0003-Introduce-Compressor-API-in-pg_dump.patch (56.8K, ../../DQn4czCWR1rcbGPLL7p3LfEr5-kGmlySm-H05VgroINdikvhtS5r9EdI6b8D8sjnbKdJ09k-cxs2AqijBeHAWk9Q8gvEAxPRHuLRhwONcGc=@pm.me/4-v16-0003-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From 4e5590699a417a080b784429f26056274bf1e142 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:03:05 +0000
Subject: [PATCH v16 3/4] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives now need to store the compression algorithm in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 390 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 9 +
src/bin/pg_dump/compress_io.c | 839 ++++++--------------------
src/bin/pg_dump/compress_io.h | 68 ++-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 102 ++--
src/bin/pg_dump/pg_backup_archiver.h | 4 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 94 +--
src/bin/pg_dump/t/002_pg_dump.pl | 6 +-
11 files changed, 777 insertions(+), 760 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..bc6d1abc77
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,390 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ int compressionLevel;
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, gzipcs->compressionLevel) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+typedef struct GzipData
+{
+ gzFile fp;
+ int compressionLevel;
+} GzipData;
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ size_t ret;
+
+ ret = gzread(gd->fp, ptr, size);
+ if (ret != size && !gzeof(gd->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gd->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzwrite(gd->fp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gd->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gd->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzgets(gd->fp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gd->fp);
+
+ save_errno = errno;
+ free(gd);
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+
+ return gzeof(gd->fp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gd->fp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
+{
+ GzipData *gd = (GzipData *) CFH->private;
+ char mode_compression[32];
+
+ if (gd->compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, gd->compressionLevel);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gd->fp = gzdopen(dup(fd), mode_compression);
+ else
+ gd->fp = gzopen(path, mode_compression);
+
+ if (gd->fp == NULL)
+ return 1;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ GzipData *gd;
+
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ gd = pg_malloc0(sizeof(GzipData));
+ gd->compressionLevel = compressionLevel;
+
+ CFH->private = gd;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+
+void
+InitCompressGzip(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with zlib support");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..ab0362c1f3
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, int compressionLevel);
+extern void InitCompressGzip(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index cb59300cb5..e7a0e57d8b 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,84 +68,67 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_specification compression_spec;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+ free(buf);
+}
+
+
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compression_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compression_spec = compression_spec;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, cs->compression_spec.level);
-#endif
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF)
-{
switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressorGzip(cs, compression_spec.level);
break;
case PG_COMPRESSION_LZ4:
/* fallthrough */
@@ -150,33 +136,8 @@ ReadDataFromArchive(ArchiveHandle *AH,
pg_fatal("invalid compression method");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ return cs;
}
/*
@@ -185,545 +146,177 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_NONE:
- free(cs);
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ cs->end(AH, cs);
+ pg_free(cs);
}
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
+/*----------------------
+ * Compressed stream API
+ *----------------------
*/
-static void
-InitCompressorZlib(CompressorState *cs, int level)
-{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
-}
-
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
-{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
-}
-
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
+ if (filenamelen < suffixlen)
+ return 0;
- if (res == Z_STREAM_END)
- break;
- }
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
}
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+/* free() without changing errno; useful in several places below */
+ static void
+free_keep_errno(void *p)
{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
+ int save_errno = errno;
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
-{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
+ free(p);
+ errno = save_errno;
}
-#endif /* HAVE_LIBZ */
-
/*
- * Functions for uncompressed output.
+ * Compression None implementation
*/
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
+ static size_t
+_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
- size_t cnt;
- char *buf;
- size_t buflen;
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
+ if (size == 0)
+ return 0;
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
- free(buf);
+ return ret;
}
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+static size_t
+_write(const void *ptr, size_t size, CompressFileHandle * CFH)
{
- cs->writeF(AH, data, dLen);
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-
-/*----------------------
- * Compressed stream API
- *----------------------
- */
-
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
-{
- pg_compress_specification compression_spec;
- void *fp;
-};
-
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
-
-/* free() without changing errno; useful in several places below */
-static void
-free_keep_errno(void *p)
+static const char *
+_get_error(CompressFileHandle * CFH)
{
- int save_errno = errno;
-
- free(p);
- errno = save_errno;
+ return strerror(errno);
}
-/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_read(const char *path, const char *mode)
+static char *
+_gets(char *ptr, int size, CompressFileHandle * CFH)
{
- cfp *fp;
-
- pg_compress_specification compression_spec = {0};
-
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- {
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(path, mode, compression_spec);
- }
- else
-#endif
- {
- compression_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compression_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
-
- fname = psprintf("%s.gz", path);
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static int
+_getc(CompressFileHandle * CFH)
{
- cfp *fp;
+ FILE *fp = (FILE *) CFH->private;
+ int ret;
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compression_spec);
- else
+ ret = fgetc(fp);
+ if (ret == EOF)
{
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
- return fp;
+
+ return ret;
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_specification compression_spec)
+static int
+_close(CompressFileHandle * CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ FILE *fp = (FILE *) CFH->private;
+ int ret = 0;
- fp->compression_spec = compression_spec;
+ CFH->private = NULL;
- switch (compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp)
+ ret = fclose(fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return fp;
+ return ret;
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
-{
- return cfopen_internal(path, -1, mode, compression_spec);
-}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compression_spec)
+static int
+_eof(CompressFileHandle * CFH)
{
- return cfopen_internal(NULL, fd, mode, compression_spec);
+ return feof((FILE *) CFH->private);
}
-int
-cfread(void *ptr, int size, cfp *fp)
+static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle * CFH)
{
- int ret = 0;
+ Assert(CFH->private == NULL);
- if (size == 0)
- return 0;
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return ret;
+ return 0;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+static int
+_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
{
- int ret = 0;
+ Assert(CFH->private == NULL);
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- return ret;
+ return 0;
}
-int
-cfgetc(cfp *fp)
+static void
+InitCompressNone(CompressFileHandle * CFH)
{
- int ret = 0;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
-
- return ret;
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compression_spec)
{
- char *ret = NULL;
+ CompressFileHandle *CFH;
- switch (fp->compression_spec.algorithm)
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
+
+ switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
-
+ InitCompressNone(CFH);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("not built with zlib support");
-#endif
+ InitCompressGzip(CFH, compression_spec.level);
break;
case PG_COMPRESSION_LZ4:
/* fallthrough */
@@ -732,107 +325,77 @@ cfgets(cfp *fp, char *buf, int len)
break;
}
- return ret;
+ return CFH;
}
-int
-cfclose(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret = 0;
-
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("not built with zlib support");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
- }
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compression_spec = {0};
- free_keep_errno(fp);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
- return ret;
-}
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
-int
-cfeof(cfp *fp)
-{
- int ret = 0;
+ fname = strdup(path);
- switch (fp->compression_spec.algorithm)
+ if (hasSuffix(fname, ".gz"))
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ bool exists;
- break;
- case PG_COMPRESSION_GZIP:
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("not built with zlib support");
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
- break;
- case PG_COMPRESSION_LZ4:
- /* fallthrough */
- case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
- break;
}
- return ret;
-}
-
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ CFH = InitCompressFileHandle(compression_spec);
+ if (CFH->open(fname, -1, mode, CFH))
{
-#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("not built with zlib support");
-#endif
+ free_keep_errno(CFH);
+ CFH = NULL;
}
+ free_keep_errno(fname);
- return strerror(errno);
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle * CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index 6fad6c2cd5..1118b7a638 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,60 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle * cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle * CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle * CFH);
+ int (*getc) (CompressFileHandle * CFH);
+ int (*eof) (CompressFileHandle * CFH);
+ int (*close) (CompressFileHandle * CFH);
+ const char *(*get_error) (CompressFileHandle * CFH);
+
+ void *private;
+};
-typedef struct cfp cfp;
+extern CompressFileHandle * InitCompressFileHandle(const pg_compress_specification compress_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compression_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle * InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle * CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index d96e566846..0c73a4707e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index fb94317ad9..dbd698027c 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle * SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1127,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1143,9 +1143,10 @@ PrintTOCSummary(Archive *AHX)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression_spec.level);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1502,6 +1503,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1526,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compression_spec);
- else
- AH->OF = cfopen(filename, mode, compression_spec);
+ CFH = InitCompressFileHandle(compression_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle * savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1690,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2036,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2078,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2181,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2237,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,12 +3653,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- /*
- * For now the compression type is implied by the level. This will need
- * to change once support for more compression algorithms is added,
- * requiring a format bump.
- */
- WriteInt(AH, AH->compression_spec.level);
+ AH->WriteBytePtr(AH, AH->compression_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3722,10 +3724,11 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
- /* Guess the compression method based on the level */
- AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compression_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
+ /* Guess the compression method based on the level */
if (AH->version < K_VERS_1_4)
AH->compression_spec.level = AH->ReadBytePtr(AH);
else
@@ -3737,10 +3740,17 @@ ReadHead(ArchiveHandle *AH)
else
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("archive is compressed, but this installation does not support compression");
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
+ if (unsupported)
+ pg_fatal("archive is compressed, but this installation does not support compression");
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 4725e49747..9e97e871f0 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,12 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index d1e54644a9..512ab043af 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compression_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index f6aee775eb..4182718b0a 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,8 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
-
- cfp *LOsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *dataFH; /* currently open data file */
+ CompressFileHandle *LOsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +197,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +217,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +326,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compression_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +345,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +370,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +385,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +434,7 @@ _LoadLOs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +442,14 @@ _LoadLOs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->LOsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->LOsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the LOs TOC file line-by-line, and process each LO */
- while ((cfgets(ctx->LOsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char lofname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +464,11 @@ _LoadLOs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreLO(AH, oid);
}
- if (!cfeof(ctx->LOsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->LOsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +488,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +513,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +526,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +548,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +577,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +588,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compression_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +602,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +653,8 @@ _StartLOs(ArchiveHandle *AH, TocEntry *te)
/* The LO TOC file is never compressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->LOsTocFH = cfopen_write(fname, "ab", compression_spec);
- if (ctx->LOsTocFH == NULL)
+ ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
+ if (ctx->LOsTocFH->open_write(fname, "ab", ctx->LOsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +671,8 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,18 +685,19 @@ static void
_EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->LOsTocFH;
char buf[50];
int len;
- /* Close the LO data file itself */
- if (cfclose(ctx->dataFH) != 0)
- pg_fatal("could not close LO data file: %m");
+ /* Close the BLOB data file itself */
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
+ pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the LO in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->LOsTocFH) != len)
- pg_fatal("could not write to LOs TOC file");
+ if (CFH->write(buf, len, CFH) != len)
+ pg_fatal("could not write to blobs TOC file");
}
/*
@@ -706,8 +710,8 @@ _EndLOs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->LOsTocFH) != 0)
- pg_fatal("could not close LOs TOC file: %m");
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
+ pg_fatal("could not close blobs TOC file: %m");
ctx->LOsTocFH = NULL;
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 248540db8c..e8246a3d4c 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -91,7 +91,7 @@ my %pgdump_runs = (
'pg_restore',
'-l', "$tempdir/compression_gzip_custom.dump",
],
- expected => qr/Compression: 1/,
+ expected => qr/Compression: gzip/,
name => 'data content is gzip compressed'
},
},
@@ -235,8 +235,8 @@ my %pgdump_runs = (
'-l', "$tempdir/defaults_custom_format.dump",
],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip compressed by default if available'
},
},
--
2.34.1
[text/x-patch] v16-0004-Add-LZ4-compression-in-pg_-dump-restore.patch (28.2K, ../../DQn4czCWR1rcbGPLL7p3LfEr5-kGmlySm-H05VgroINdikvhtS5r9EdI6b8D8sjnbKdJ09k-cxs2AqijBeHAWk9Q8gvEAxPRHuLRhwONcGc=@pm.me/5-v16-0004-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 53904d54638ccc3dfbbcc844d97b8f2900c083a8 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Fri, 2 Dec 2022 16:03:12 +0000
Subject: [PATCH v16 4/4] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 13 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 43 +-
src/bin/pg_dump/compress_lz4.c | 601 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 9 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 5 +-
src/bin/pg_dump/t/002_pg_dump.pl | 82 +++-
9 files changed, 748 insertions(+), 29 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2c938cd7e1..49d218905f 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -330,9 +330,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -654,7 +655,7 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression.
+ <literal>lz4</literal> or <literal>none</literal> for no compression.
A compression detail string can optionally be specified. If the
detail string is an integer, it specifies the compression level.
Otherwise, it should be a comma-separated list of items, each of the
@@ -675,8 +676,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index e7a0e57d8b..a1058ff2fe 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -131,7 +134,8 @@ AllocateCompressor(const pg_compress_specification compression_spec,
InitCompressorGzip(cs, compression_spec.level);
break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ InitCompressorLZ4(cs, compression_spec.level);
+ break;
case PG_COMPRESSION_ZSTD:
pg_fatal("invalid compression method");
break;
@@ -182,7 +186,8 @@ free_keep_errno(void *p)
/*
* Compression None implementation
*/
- static size_t
+
+static size_t
_read(void *ptr, size_t size, CompressFileHandle * CFH)
{
FILE *fp = (FILE *) CFH->private;
@@ -319,7 +324,8 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
InitCompressGzip(CFH, compression_spec.level);
break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ InitCompressLZ4(CFH, compression_spec.level);
+ break;
case PG_COMPRESSION_ZSTD:
pg_fatal("invalid compression method");
break;
@@ -332,12 +338,12 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -373,6 +379,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compression_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..8f93f05e87
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,601 @@
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*
+ * LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
+ * Redefine it for installations with a lesser version.
+ */
+#ifndef LZ4F_HEADER_SIZE_MAX
+#define LZ4F_HEADER_SIZE_MAX 32
+#endif
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle * CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle * CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle * CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (compressionLevel >= 0)
+ lz4fp->prefs.compressionLevel = compressionLevel;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+
+void
+InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel)
+{
+ pg_fatal("not built with LZ4 support");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..fbec9a508d
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,9 @@
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, int compressionLevel);
+extern void InitCompressLZ4(CompressFileHandle * CFH, int compressionLevel);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 0c73a4707e..b27e92ffd0 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -83,7 +84,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index dbd698027c..9bf64b7fa9 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -395,6 +395,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2074,7 +2078,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2084,6 +2088,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3747,6 +3755,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
pg_fatal("archive is compressed, but this installation does not support compression");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index ad6693c358..40f4949a9f 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -715,13 +715,12 @@ main(int argc, char **argv)
case PG_COMPRESSION_NONE:
/* fallthrough */
case PG_COMPRESSION_GZIP:
+ /* fallthrough */
+ case PG_COMPRESSION_LZ4:
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
}
/*
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index e8246a3d4c..20f8a8c539 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -136,6 +136,80 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=lz4', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ command_like => {
+ command => [
+ 'pg_restore',
+ '-l', "$tempdir/compression_lz4_custom.dump",
+ ],
+ expected => qr/Compression: lz4/,
+ name => 'data content is lz4 compressed'
+ },
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ # Verify that data files where compressed
+ glob_patterns => [
+ "$tempdir/compression_lz4_dir/toc.dat",
+ "$tempdir/compression_lz4_dir/*.dat.lz4",
+ ],
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4163,11 +4237,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-06 00:22 ` Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Michael Paquier @ 2022-12-06 00:22 UTC (permalink / raw)
To: [email protected]; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Mon, Dec 05, 2022 at 12:48:28PM +0000, [email protected] wrote:
> I also took the liberty of applying the test pattern when it the dump
> is explicitly compressed.
Sticking with glob_patterns is fine by me.
> I was thinking a bit more about this. I think that we can use the list
> TOC option of pg_restore. This option will first print out the header
> info which contains the compression. Perl utils already support to
> parse the generated output of a command. Please find an attempt to do
> so in the attached. The benefits of having some testing for this case
> become a bit more obvious in 0004 of the patchset, when lz4 is
> introduced.
This is where the fun is. What you are doing here is more complete,
and we would make sure that the custom and data directory would always
see their contents compressed by default. And it would have caught
the bug you mentioned upthread for the custom format.
I have kept things as you proposed at the end, added a few comments,
documented the new command_like and an extra command_like for
defaults_dir_format. Glad to see this addressed, thanks!
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../Y46LYQZltv7dx%[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-06 15:52 ` [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-06 15:52 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Tuesday, December 6th, 2022 at 1:22 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Mon, Dec 05, 2022 at 12:48:28PM +0000, [email protected] wrote:
>
> This is where the fun is. What you are doing here is more complete,
> and we would make sure that the custom and data directory would always
> see their contents compressed by default. And it would have caught
> the bug you mentioned upthread for the custom format.
Thank you very much Michael.
> I have kept things as you proposed at the end, added a few comments,
> documented the new command_like and an extra command_like for
> defaults_dir_format. Glad to see this addressed, thanks!
Please find attached v17, which builds on top of what is already
committed. I dare to think 0001 as ready to be reviewed. 0002 is
also complete albeit with some documentation gaps.
Cheers,
//Georgios
> --
> Michael
Attachments:
[text/x-patch] v17-0001-Prepare-pg_dump-internals-for-additional-compres.patch (22.0K, ../../4wfmZ9BuGbkM6JXggut4g_gPl1hLbVeUwPEQh5nGgzlKCgnbtRaw7-nPcyPelGdG8ttH9A7vgyX_Z2fwZxuqHt1xENg9mqDfP_94qoZ5WxQ=@pm.me/2-v17-0001-Prepare-pg_dump-internals-for-additional-compres.patch)
download | inline diff:
From 859a9ca65ddafaf98854b81639d310c11c5cae9c Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 6 Dec 2022 15:42:03 +0000
Subject: [PATCH v17 1/3] Prepare pg_dump internals for additional compression
methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about it and is using it throughout.
---
src/bin/pg_dump/compress_io.c | 396 +++++++++++++++++++--------
src/bin/pg_dump/pg_backup_archiver.c | 128 +++------
src/bin/pg_dump/pg_backup_archiver.h | 27 +-
3 files changed, 323 insertions(+), 228 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index a7df600cc0..b893aca1ed 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -101,7 +105,7 @@ AllocateCompressor(const pg_compress_specification compression_spec,
#ifndef HAVE_LIBZ
if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
@@ -128,15 +132,25 @@ ReadDataFromArchive(ArchiveHandle *AH,
const pg_compress_specification compression_spec,
ReadFunc readF)
{
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ switch (compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
}
@@ -149,20 +163,22 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ WriteDataToArchiveNone(AH, cs, data, dLen);
+ break;
case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
}
}
@@ -173,10 +189,26 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
free(cs);
}
@@ -391,10 +423,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_specification compression_spec;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -490,127 +520,203 @@ cfopen_write(const char *path, const char *mode,
}
/*
- * Opens file 'path' in 'mode'. If compression is GZIP, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ fp->compression_spec = compression_spec;
+
+ switch (compression_spec.algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compression_spec.level);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(path, -1, mode, compression_spec);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(NULL, fd, mode, compression_spec);
+}
int
cfread(void *ptr, int size, cfp *fp)
{
- int ret;
+ int ret = 0;
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
int
cfgetc(cfp *fp)
{
- int ret;
+ int ret = 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
return ret;
@@ -619,65 +725,119 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret = NULL;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret = 0;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compression_spec.algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7f7a0f1ce7..fb94317ad9 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -101,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -277,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -362,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -391,17 +383,24 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -1128,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
@@ -1503,58 +1502,32 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression_spec.level);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compression_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compression_spec);
if (!AH->OF)
{
@@ -1565,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename,
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1715,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2219,6 +2178,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2272,8 +2232,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index f72446ed5b..4725e49747 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
--
2.34.1
[text/x-patch] v17-0002-Introduce-Compressor-API-in-pg_dump.patch (62.2K, ../../4wfmZ9BuGbkM6JXggut4g_gPl1hLbVeUwPEQh5nGgzlKCgnbtRaw7-nPcyPelGdG8ttH9A7vgyX_Z2fwZxuqHt1xENg9mqDfP_94qoZ5WxQ=@pm.me/3-v17-0002-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From f7915e5dcbb44f323f13d33213723490174a3c1a Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 6 Dec 2022 15:42:11 +0000
Subject: [PATCH v17 2/3] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives now need to store the compression algorithm in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 394 +++++++++++
src/bin/pg_dump/compress_gzip.h | 22 +
src/bin/pg_dump/compress_io.c | 897 +++++++-------------------
src/bin/pg_dump/compress_io.h | 71 +-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 102 +--
src/bin/pg_dump/pg_backup_archiver.h | 5 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 94 +--
src/bin/pg_dump/t/002_pg_dump.pl | 10 +-
src/tools/pgindent/typedefs.list | 2 +
12 files changed, 823 insertions(+), 799 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..6a89deaa4b
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,394 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_gzip.c
+ * Routines for archivers to write an uncompressed or compressed data
+ * stream.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_gzip.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_gzip.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, cs->compression_spec.level) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+
+ cs->private = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+ size_t ret;
+
+ ret = gzread(gzfp, ptr, size);
+ if (ret != size && !gzeof(gzfp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gzfp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+
+ return gzwrite(gzfp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gzfp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gzfp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+
+ return gzgets(gzfp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+ int save_errno;
+ int ret;
+
+ CFH->private = NULL;
+
+ ret = gzclose(gzfp);
+
+ save_errno = errno;
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+
+ return gzeof(gzfp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gzfp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
+{
+ gzFile gzfp;
+ char mode_compression[32];
+
+ if (CFH->compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, CFH->compression_spec.level);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gzfp = gzdopen(dup(fd), mode_compression);
+ else
+ gzfp = gzopen(path, mode_compression);
+
+ if (gzfp == NULL)
+ return 1;
+
+ CFH->private = gzfp;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ CFH->open = Gzip_open;
+ CFH->open_write = Gzip_open_write;
+ CFH->read = Gzip_read;
+ CFH->write = Gzip_write;
+ CFH->gets = Gzip_gets;
+ CFH->getc = Gzip_getc;
+ CFH->close = Gzip_close;
+ CFH->eof = Gzip_eof;
+ CFH->get_error = Gzip_get_error;
+
+ CFH->compression_spec = compression_spec;
+
+ CFH->private = NULL;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "gzip");
+}
+
+void
+InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "gzip");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..6dfd0eb04d
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_gzip.h
+ * Interface to compress_io.c routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_gzip.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec);
+extern void InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index b893aca1ed..c33736dd49 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -9,30 +9,30 @@
*
* This file includes two APIs for dealing with compressed data. The first
* provides more flexibility, using callbacks to read/write data from the
- * underlying stream. The second API is a wrapper around fopen/gzopen and
+ * underlying stream. The second API is a wrapper around fopen and
* friends, providing an interface similar to those, but abstracts away
- * the possible compression. Both APIs use libz for the compression, but
- * the second API uses gzip headers, so the resulting files can be easily
- * manipulated with the gzip utility.
+ * the possible compression. The second API is aimed for the resulting
+ * files can be easily manipulated with an external compression utility
+ * program.
*
* Compressor API
* --------------
*
* The interface for writing to an archive consists of three functions:
- * AllocateCompressor, WriteDataToArchive and EndCompressor. First you call
- * AllocateCompressor, then write all the data by calling WriteDataToArchive
- * as many times as needed, and finally EndCompressor. WriteDataToArchive
- * and EndCompressor will call the WriteFunc that was provided to
- * AllocateCompressor for each chunk of compressed data.
+ * AllocateCompressor, writeData, and EndCompressor. First you call
+ * AllocateCompressor, then write all the data by calling writeData as many
+ * times as needed, and finally EndCompressor. writeData will call the
+ * WriteFunc that was provided to AllocateCompressor for each chunk of
+ * compressed data.
*
- * The interface for reading an archive consists of just one function:
- * ReadDataFromArchive. ReadDataFromArchive reads the whole compressed input
- * stream, by repeatedly calling the given ReadFunc. ReadFunc returns the
- * compressed data chunk at a time, and ReadDataFromArchive decompresses it
- * and passes the decompressed data to ahwrite(), until ReadFunc returns 0
- * to signal EOF.
- *
- * The interface is the same for compressed and uncompressed streams.
+ * The interface for reading an archive consists of the same three functions:
+ * AllocateCompressor, readData, and EndCompressor. First you call
+ * AllocateCompressor, then read all the data by calling readData to read the
+ * whole compressed stream which repeatedly calls the given ReadFunc. ReadFunc
+ * returns the compressed data chunk at a time, and readData decompresses it
+ * and passes the decompressed data to ahwrite(), until ReadFunc returns 0 to
+ * signal EOF. The interface is the same for compressed and uncompressed
+ * streams.
*
* Compressed stream API
* ----------------------
@@ -51,9 +51,12 @@
*
*-------------------------------------------------------------------------
*/
+#include <sys/stat.h>
+#include <unistd.h>
#include "postgres_fe.h"
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,85 +68,70 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_specification compression_spec;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
+
+ free(buf);
+}
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs,
+ const pg_compress_specification compression_spec)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+
+ cs->compression_spec = compression_spec;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compression_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compression_spec = compression_spec;
-
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, cs->compression_spec.level);
-#endif
-
- return cs;
-}
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF)
-{
switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs, compression_spec);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
+ InitCompressorGzip(cs, compression_spec);
break;
case PG_COMPRESSION_LZ4:
pg_fatal("compression with %s is not yet supported", "LZ4");
@@ -152,35 +140,8 @@ ReadDataFromArchive(ArchiveHandle *AH,
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ return cs;
}
/*
@@ -189,247 +150,28 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- free(cs);
-}
-
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
- */
-
-static void
-InitCompressorZlib(CompressorState *cs, int level)
-{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
-}
-
-static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
-{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
-
- free(cs->zlibOut);
- free(cs->zp);
-}
-
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
-{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
-
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
-
- if (res == Z_STREAM_END)
- break;
- }
-}
-
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
-}
-
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
-{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
-
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
-
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
-
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
-}
-#endif /* HAVE_LIBZ */
-
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
-{
- size_t cnt;
- char *buf;
- size_t buflen;
-
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
-
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
-
- free(buf);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
-{
- cs->writeF(AH, data, dLen);
-}
-
-
/*----------------------
* Compressed stream API
*----------------------
*/
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- pg_compress_specification compression_spec;
- void *fp;
-};
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
+}
/* free() without changing errno; useful in several places below */
static void
@@ -442,418 +184,223 @@ free_keep_errno(void *p)
}
/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
+ * Compression None implementation
*/
-cfp *
-cfopen_read(const char *path, const char *mode)
+static size_t
+_read(void *ptr, size_t size, CompressFileHandle *CFH)
{
- cfp *fp;
-
- pg_compress_specification compression_spec = {0};
-
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- {
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(path, mode, compression_spec);
- }
- else
-#endif
- {
- compression_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compression_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
+ FILE *fp = (FILE *) CFH->private;
+ size_t ret;
- fname = psprintf("%s.gz", path);
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
-}
+ if (size == 0)
+ return 0;
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
-{
- cfp *fp;
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compression_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ return ret;
}
-/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
- */
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_specification compression_spec)
+static size_t
+_write(const void *ptr, size_t size, CompressFileHandle *CFH)
{
- cfp *fp = pg_malloc(sizeof(cfp));
-
- fp->compression_spec = compression_spec;
-
- switch (compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return fp;
+ return fwrite(ptr, 1, size, (FILE *) CFH->private);
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static const char *
+_get_error(CompressFileHandle *CFH)
{
- return cfopen_internal(path, -1, mode, compression_spec);
+ return strerror(errno);
}
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compression_spec)
+static char *
+_gets(char *ptr, int size, CompressFileHandle *CFH)
{
- return cfopen_internal(NULL, fd, mode, compression_spec);
+ return fgets(ptr, size, (FILE *) CFH->private);
}
-int
-cfread(void *ptr, int size, cfp *fp)
+static int
+_getc(CompressFileHandle *CFH)
{
- int ret = 0;
-
- if (size == 0)
- return 0;
+ FILE *fp = (FILE *) CFH->private;
+ int ret;
- switch (fp->compression_spec.algorithm)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, fp->fp);
- if (ret != size && !feof(fp->fp))
- READ_ERROR_EXIT(fp->fp);
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzread(fp->fp, ptr, size);
- if (ret != size && !gzeof(fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
return ret;
}
-int
-cfwrite(const void *ptr, int size, cfp *fp)
+static int
+_close(CompressFileHandle *CFH)
{
+ FILE *fp = (FILE *) CFH->private;
int ret = 0;
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite(fp->fp, ptr, size);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ CFH->private = NULL;
+
+ if (fp)
+ ret = fclose(fp);
return ret;
}
-int
-cfgetc(cfp *fp)
+
+static int
+_eof(CompressFileHandle *CFH)
{
- int ret = 0;
+ return feof((FILE *) CFH->private);
+}
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc(fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->fp);
+static int
+_open(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
+{
+ Assert(CFH->private == NULL);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof(fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ if (fd >= 0)
+ CFH->private = fdopen(dup(fd), mode);
+ else
+ CFH->private = fopen(path, mode);
- return ret;
+ if (CFH->private == NULL)
+ return 1;
+
+ return 0;
}
-char *
-cfgets(cfp *fp, char *buf, int len)
+static int
+_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
{
- char *ret = NULL;
+ Assert(CFH->private == NULL);
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, fp->fp);
+ CFH->private = fopen(path, mode);
+ if (CFH->private == NULL)
+ return 1;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets(fp->fp, buf, len);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ return 0;
+}
- return ret;
+static void
+InitCompressNone(CompressFileHandle *CFH,
+ const pg_compress_specification compression_spec)
+{
+ CFH->open = _open;
+ CFH->open_write = _open_write;
+ CFH->read = _read;
+ CFH->write = _write;
+ CFH->gets = _gets;
+ CFH->getc = _getc;
+ CFH->close = _close;
+ CFH->eof = _eof;
+ CFH->get_error = _get_error;
+
+ CFH->private = NULL;
}
-int
-cfclose(cfp *fp)
+/*
+ * Public interface
+ */
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compression_spec)
{
- int ret = 0;
+ CompressFileHandle *CFH;
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
- }
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
- switch (fp->compression_spec.algorithm)
+ switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ret = fclose(fp->fp);
- fp->fp = NULL;
-
+ InitCompressNone(CFH, compression_spec);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose(fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
+ InitCompressGzip(CFH, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
+ /* fallthrough */
case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
+ pg_fatal("invalid compression method");
break;
}
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
-int
-cfeof(cfp *fp)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ *
+ * On failure, return NULL with an error code in errno.
+ *
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- int ret = 0;
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compression_spec = {0};
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof(fp->fp);
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof(fp->fp);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- return ret;
-}
+ fname = strdup(path);
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ if (hasSuffix(fname, ".gz"))
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
+ bool exists;
+
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror(fp->fp, &errnum);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
}
- return strerror(errno);
+ CFH = InitCompressFileHandle(compression_spec);
+ if (CFH->open(fname, -1, mode, CFH))
+ {
+ free_keep_errno(CFH);
+ CFH = NULL;
+ }
+ free_keep_errno(fname);
+
+ return CFH;
}
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
+int
+DestroyCompressFileHandle(CompressFileHandle *CFH)
{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
+ int ret = 0;
- if (filenamelen < suffixlen)
- return 0;
+ if (CFH->private)
+ ret = CFH->close(CFH);
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
+ free_keep_errno(CFH);
-#endif
+ return ret;
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index 6fad6c2cd5..3053dc43dd 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,63 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ pg_compress_specification compression_spec;
+ void *private;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open) (const char *path, int fd, const char *mode,
+ CompressFileHandle *CFH);
+ int (*open_write) (const char *path, const char *mode,
+ CompressFileHandle *cxt);
+ size_t (*read) (void *ptr, size_t size, CompressFileHandle *CFH);
+ size_t (*write) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets) (char *s, int size, CompressFileHandle *CFH);
+ int (*getc) (CompressFileHandle *CFH);
+ int (*eof) (CompressFileHandle *CFH);
+ int (*close) (CompressFileHandle *CFH);
+ const char *(*get_error) (CompressFileHandle *CFH);
+
+ pg_compress_specification compression_spec;
+ void *private;
+};
-typedef struct cfp cfp;
+extern CompressFileHandle *InitCompressFileHandle(
+ const pg_compress_specification compression_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compression_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle *InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle *CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index d96e566846..0c73a4707e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index fb94317ad9..1a3b309484 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1127,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1143,9 +1143,10 @@ PrintTOCSummary(Archive *AHX)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression_spec.level);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1502,6 +1503,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1526,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compression_spec);
- else
- AH->OF = cfopen(filename, mode, compression_spec);
+ CFH = InitCompressFileHandle(compression_spec);
- if (!AH->OF)
+ if (CFH->open(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1690,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2036,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2078,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2181,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2237,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,12 +3653,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- /*
- * For now the compression type is implied by the level. This will need
- * to change once support for more compression algorithms is added,
- * requiring a format bump.
- */
- WriteInt(AH, AH->compression_spec.level);
+ AH->WriteBytePtr(AH, AH->compression_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3722,10 +3724,11 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
- /* Guess the compression method based on the level */
- AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compression_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
+ /* Guess the compression method based on the level */
if (AH->version < K_VERS_1_4)
AH->compression_spec.level = AH->ReadBytePtr(AH);
else
@@ -3737,10 +3740,17 @@ ReadHead(ArchiveHandle *AH)
else
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("archive is compressed, but this installation does not support compression");
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
+ if (unsupported)
+ pg_fatal("archive is compressed, but this installation does not support compression");
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 4725e49747..18b38c17ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,13 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add
+ * compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index d1e54644a9..512ab043af 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compression_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index f6aee775eb..4182718b0a 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,8 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
-
- cfp *LOsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *dataFH; /* currently open data file */
+ CompressFileHandle *LOsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +197,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +217,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +326,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compression_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +345,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -370,7 +370,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +385,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +434,7 @@ _LoadLOs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +442,14 @@ _LoadLOs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->LOsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->LOsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the LOs TOC file line-by-line, and process each LO */
- while ((cfgets(ctx->LOsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets(line, MAXPGPATH, CFH)) != NULL)
{
char lofname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +464,11 @@ _LoadLOs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreLO(AH, oid);
}
- if (!cfeof(ctx->LOsTocFH))
+ if (!CFH->eof(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->LOsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +488,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
return 1;
@@ -512,8 +513,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc(CFH);
}
/*
@@ -524,15 +526,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error(CFH));
}
}
@@ -545,12 +548,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +577,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +588,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compression_spec);
+ if (tocFH->open_write(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +602,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +653,8 @@ _StartLOs(ArchiveHandle *AH, TocEntry *te)
/* The LO TOC file is never compressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->LOsTocFH = cfopen_write(fname, "ab", compression_spec);
- if (ctx->LOsTocFH == NULL)
+ ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
+ if (ctx->LOsTocFH->open_write(fname, "ab", ctx->LOsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +671,8 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+ if (ctx->dataFH->open_write(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,18 +685,19 @@ static void
_EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->LOsTocFH;
char buf[50];
int len;
- /* Close the LO data file itself */
- if (cfclose(ctx->dataFH) != 0)
- pg_fatal("could not close LO data file: %m");
+ /* Close the BLOB data file itself */
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
+ pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the LO in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->LOsTocFH) != len)
- pg_fatal("could not write to LOs TOC file");
+ if (CFH->write(buf, len, CFH) != len)
+ pg_fatal("could not write to blobs TOC file");
}
/*
@@ -706,8 +710,8 @@ _EndLOs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->LOsTocFH) != 0)
- pg_fatal("could not close LOs TOC file: %m");
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
+ pg_fatal("could not close blobs TOC file: %m");
ctx->LOsTocFH = NULL;
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 6656222363..22a7c5c37c 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -94,7 +94,7 @@ my %pgdump_runs = (
command => [
'pg_restore', '-l', "$tempdir/compression_gzip_custom.dump",
],
- expected => qr/Compression: 1/,
+ expected => qr/Compression: gzip/,
name => 'data content is gzip-compressed'
},
},
@@ -239,8 +239,8 @@ my %pgdump_runs = (
command =>
[ 'pg_restore', '-l', "$tempdir/defaults_custom_format.dump", ],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip-compressed by default if available',
},
},
@@ -264,8 +264,8 @@ my %pgdump_runs = (
command =>
[ 'pg_restore', '-l', "$tempdir/defaults_dir_format", ],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip-compressed by default',
},
glob_patterns => [
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 58daeca831..29266b5de9 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -428,6 +428,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
+CompressFileHandle
CompressionLocation
CompressorState
ComputeXidHorizonsResult
@@ -1033,6 +1034,7 @@ GucStack
GucStackState
GucStringAssignHook
GucStringCheckHook
+GzipCompressorState
HANDLE
HASHACTION
HASHBUCKET
--
2.34.1
[text/x-patch] v17-0003-Add-LZ4-compression-in-pg_-dump-restore.patch (29.6K, ../../4wfmZ9BuGbkM6JXggut4g_gPl1hLbVeUwPEQh5nGgzlKCgnbtRaw7-nPcyPelGdG8ttH9A7vgyX_Z2fwZxuqHt1xENg9mqDfP_94qoZ5WxQ=@pm.me/4-v17-0003-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From f20ba2ccb17ab5f8e75f9b73df03e3186f4d6024 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Tue, 6 Dec 2022 15:42:31 +0000
Subject: [PATCH v17 3/3] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 13 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 39 +-
src/bin/pg_dump/compress_lz4.c | 618 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 22 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 5 +-
src/bin/pg_dump/t/002_pg_dump.pl | 82 +++-
src/tools/pgindent/typedefs.list | 1 +
10 files changed, 776 insertions(+), 28 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2c938cd7e1..49d218905f 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -330,9 +330,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -654,7 +655,7 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression.
+ <literal>lz4</literal> or <literal>none</literal> for no compression.
A compression detail string can optionally be specified. If the
detail string is an integer, it specifies the compression level.
Otherwise, it should be a comma-separated list of items, each of the
@@ -675,8 +676,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index c33736dd49..3c2a297350 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -38,13 +38,15 @@
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -57,6 +59,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -134,7 +137,7 @@ AllocateCompressor(const pg_compress_specification compression_spec,
InitCompressorGzip(cs, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
+ InitCompressorLZ4(cs, compression_spec);
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
@@ -324,7 +327,8 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
InitCompressGzip(CFH, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ InitCompressLZ4(CFH, compression_spec);
+ break;
case PG_COMPRESSION_ZSTD:
pg_fatal("invalid compression method");
break;
@@ -337,12 +341,12 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz", trying in that order.
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
*
* On failure, return NULL with an error code in errno.
- *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
@@ -378,6 +382,17 @@ InitDiscoverCompressFileHandle(const char *path, const char *mode)
if (exists)
compression_spec.algorithm = PG_COMPRESSION_GZIP;
}
+#endif
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
+
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
}
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..25bc9108dd
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,618 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_lz4.c
+ * Routines for archivers to write an uncompressed or compressed data
+ * stream.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_lz4.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*
+ * LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
+ * Redefine it for installations with a lesser version.
+ */
+#ifndef LZ4F_HEADER_SIZE_MAX
+#define LZ4F_HEADER_SIZE_MAX 32
+#endif
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ cs->compression_spec = compression_spec;
+
+ /* Will be lazy init'd */
+ cs->private = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle *CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle *CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ LZ4File *lz4fp;
+
+ CFH->open = LZ4File_open;
+ CFH->open_write = LZ4File_open_write;
+ CFH->read = LZ4File_read;
+ CFH->write = LZ4File_write;
+ CFH->gets = LZ4File_gets;
+ CFH->getc = LZ4File_getc;
+ CFH->eof = LZ4File_eof;
+ CFH->close = LZ4File_close;
+ CFH->get_error = LZ4File_get_error;
+
+ CFH->compression_spec = compression_spec;
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (CFH->compression_spec.level >= 0)
+ lz4fp->prefs.compressionLevel = CFH->compression_spec.level;
+
+ CFH->private = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "LZ4");
+}
+
+void
+InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "LZ4");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..74595db1b9
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_lz4.h
+ * Interface to compress_io.c routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_lz4.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec);
+extern void InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 0c73a4707e..b27e92ffd0 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -83,7 +84,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1a3b309484..c0f031b052 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -395,6 +395,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2074,7 +2078,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2084,6 +2088,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3747,6 +3755,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
pg_fatal("archive is compressed, but this installation does not support compression");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index ad6693c358..40f4949a9f 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -715,13 +715,12 @@ main(int argc, char **argv)
case PG_COMPRESSION_NONE:
/* fallthrough */
case PG_COMPRESSION_GZIP:
+ /* fallthrough */
+ case PG_COMPRESSION_LZ4:
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
}
/*
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 22a7c5c37c..d1a9e1d45b 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -139,6 +139,80 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=lz4', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ command_like => {
+ command => [
+ 'pg_restore',
+ '-l', "$tempdir/compression_lz4_custom.dump",
+ ],
+ expected => qr/Compression: lz4/,
+ name => 'data content is lz4 compressed'
+ },
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ # Verify that data files where compressed
+ glob_patterns => [
+ "$tempdir/compression_lz4_dir/toc.dat",
+ "$tempdir/compression_lz4_dir/*.dat.lz4",
+ ],
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4175,11 +4249,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 29266b5de9..0e144bceaf 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1380,6 +1380,7 @@ LWLock
LWLockHandle
LWLockMode
LWLockPadded
+LZ4CompressorState
LZ4F_compressionContext_t
LZ4F_decompressOptions_t
LZ4F_decompressionContext_t
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-17 23:26 ` Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-12-17 23:26 UTC (permalink / raw)
To: [email protected]; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
001: still refers to "gzip", which is correct for -Fp and -Fd but not
for -Fc, for which it's more correct to say "zlib". That affects the
name of the function, structures, comments, etc. I'm not sure if it's
an issue to re-use the basebackup compression routines here. Maybe we
should accept "-Zn" for zlib output (-Fc), but reject "gzip:9", which
I'm sure some will find confusing, as it does not output. Maybe 001
should be split into a patch to re-use the existing "cfp" interface
(which is a clear win), and 002 to re-use the basebackup interfaces for
user input and constants, etc.
001 still doesn't compile on freebsd, and 002 doesn't compile on
windows. Have you checked test results from cirrusci on your private
github account ?
002 says:
+ save_errno = errno;
+ errno = save_errno;
I suppose that's intended to wrap the preceding library call.
002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
doesn't store the passed-in compression_spec.
003 still uses <lz4.h> and not "lz4.h".
Earlier this year I also suggested to include an 999 patch to change to
use LZ4 as the default compression, to exercise the new code under CI.
I suggest to re-open the cf patch entry after that passes tests on all
platforms and when it's ready for more review.
BTW, some of these review comments are the same as what I sent earlier
this year.
https://www.postgresql.org/message-id/20220326162156.GI28503%40telsasoft.com
https://www.postgresql.org/message-id/20220705151328.GQ13040%40telsasoft.com
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-12-19 04:06 ` Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-19 17:42 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 2 replies; 67+ messages in thread
From: Michael Paquier @ 2022-12-19 04:06 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: [email protected]; [email protected]; Rachel Heaton <[email protected]>
On Sat, Dec 17, 2022 at 05:26:15PM -0600, Justin Pryzby wrote:
> 001: still refers to "gzip", which is correct for -Fp and -Fd but not
> for -Fc, for which it's more correct to say "zlib".
Or should we begin by changing all these existing "not built with zlib
support" error strings to the more generic "this build does not
support compression with %s" to reduce the number of messages to
translate? That would bring consistency with the other tools dealing
with compression.
> That affects the
> name of the function, structures, comments, etc. I'm not sure if it's
> an issue to re-use the basebackup compression routines here. Maybe we
> should accept "-Zn" for zlib output (-Fc), but reject "gzip:9", which
> I'm sure some will find confusing, as it does not output. Maybe 001
> should be split into a patch to re-use the existing "cfp" interface
> (which is a clear win), and 002 to re-use the basebackup interfaces for
> user input and constants, etc.
>
> 001 still doesn't compile on freebsd, and 002 doesn't compile on
> windows. Have you checked test results from cirrusci on your private
> github account ?
FYI, I have re-added an entry to the CF app to get some automated
coverage:
https://commitfest.postgresql.org/41/3571/
On MinGW, a complain about the open() callback, which I guess ought to
be avoided with a rename:
[00:16:37.254] compress_gzip.c:356:38: error: macro "open" passed 4 arguments, but takes just 3
[00:16:37.254] 356 | ret = CFH->open(fname, -1, mode, CFH);
[00:16:37.254] | ^
[00:16:37.254] In file included from ../../../src/include/c.h:1309,
[00:16:37.254] from ../../../src/include/postgres_fe.h:25,
[00:16:37.254] from compress_gzip.c:15:
On MSVC, some declaration conflicts, for a similar issue:
[00:12:31.966] ../src/bin/pg_dump/compress_io.c(193): error C2371: '_read': redefinition; different basic types
[00:12:31.966] C:\Program Files (x86)\Windows Kits\10\include\10.0.20348.0\ucrt\corecrt_io.h(252): note: see declaration of '_read'
[00:12:31.966] ../src/bin/pg_dump/compress_io.c(210): error C2371: '_write': redefinition; different basic types
[00:12:31.966] C:\Program Files (x86)\Windows Kits\10\include\10.0.20348.0\ucrt\corecrt_io.h(294): note: see declaration of '_write'
> 002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
> doesn't store the passed-in compression_spec.
Hmm. This looks like a gap in the existing tests that we'd better fix
first. This CI is green on Linux.
> 003 still uses <lz4.h> and not "lz4.h".
This should be <lz4.h>, not "lz4.h".
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../Y5%[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-19 17:03 ` [email protected]
2022-12-19 17:27 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
1 sibling, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-19 17:03 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Monday, December 19th, 2022 at 5:06 AM, Michael Paquier <[email protected]> wrote:
>
>
> On Sat, Dec 17, 2022 at 05:26:15PM -0600, Justin Pryzby wrote:
>
Thank you for the comments, please find v18 attached.
> > 001: still refers to "gzip", which is correct for -Fp and -Fd but not
> > for -Fc, for which it's more correct to say "zlib".
>
>
> Or should we begin by changing all these existing "not built with zlib
> support" error strings to the more generic "this build does not
> support compression with %s" to reduce the number of messages to
> translate? That would bring consistency with the other tools dealing
> with compression.
This has been the approach from 0002 on-wards. In the attached it is also
applied on the remaining location in 0001.
>
> > That affects the
> > name of the function, structures, comments, etc. I'm not sure if it's
> > an issue to re-use the basebackup compression routines here. Maybe we
> > should accept "-Zn" for zlib output (-Fc), but reject "gzip:9", which
> > I'm sure some will find confusing, as it does not output. Maybe 001
> > should be split into a patch to re-use the existing "cfp" interface
> > (which is a clear win), and 002 to re-use the basebackup interfaces for
> > user input and constants, etc.
> >
> > 001 still doesn't compile on freebsd, and 002 doesn't compile on
> > windows. Have you checked test results from cirrusci on your private
> > github account ?
There are still known gaps in 0002 and 0003, for example documentation,
and I have not been focusing too much on those. You are right, it is helpful
and kind to try to reduce the noise. The attached should have hopefully
tackled the ci errors.
>
> FYI, I have re-added an entry to the CF app to get some automated
> coverage:
> https://commitfest.postgresql.org/41/3571/
Much obliged. Should I change the state to "ready for review" when post a
new version or should I leave that to the senior personnel?
>
> On MinGW, a complain about the open() callback, which I guess ought to
> be avoided with a rename:
> [00:16:37.254] compress_gzip.c:356:38: error: macro "open" passed 4 arguments, but takes just 3
> [00:16:37.254] 356 | ret = CFH->open(fname, -1, mode, CFH);
>
> [00:16:37.254] | ^
> [00:16:37.254] In file included from ../../../src/include/c.h:1309,
> [00:16:37.254] from ../../../src/include/postgres_fe.h:25,
> [00:16:37.254] from compress_gzip.c:15:
>
> On MSVC, some declaration conflicts, for a similar issue:
> [00:12:31.966] ../src/bin/pg_dump/compress_io.c(193): error C2371: '_read': redefinition; different basic types
> [00:12:31.966] C:\Program Files (x86)\Windows Kits\10\include\10.0.20348.0\ucrt\corecrt_io.h(252): note: see declaration of '_read'
> [00:12:31.966] ../src/bin/pg_dump/compress_io.c(210): error C2371: '_write': redefinition; different basic types
> [00:12:31.966] C:\Program Files (x86)\Windows Kits\10\include\10.0.20348.0\ucrt\corecrt_io.h(294): note: see declaration of '_write'
>
A rename was enough.
> > 002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
> > doesn't store the passed-in compression_spec.
>
I am afraid I have not been able to reproduce this error. I tried both
debian and freebsd after I addressed the compilation warnings. Which
error did you get? Is it still present in the attached?
> Hmm. This looks like a gap in the existing tests that we'd better fix
> first. This CI is green on Linux.
As the code stands, the compression level is not stored in the custom
format's header as it is no longer relevant information. We can decide
to make it relevant for the tests only on the expense of increasing
dump size by four bytes. In either case this is not applicable in
current head and can wait for 0002's turn.
Cheers,
//Georgios
>
> > 003 still uses <lz4.h> and not "lz4.h".
>
>
> This should be <lz4.h>, not "lz4.h".
>
> --
> Michael
Attachments:
[text/x-patch] v18-0001-Prepare-pg_dump-internals-for-additional-compres.patch (22.5K, ../../IZzs7Z2ny5hRK1EhTynSg3DnCo_z-UvEJN4Q6Hf0CUHWx3BcNwt-2z5vvvvA-Z8PQL-AYiL9YQPlKylATZ_Z6HnJ55YZUHgL66H9MZQFi48=@pm.me/2-v18-0001-Prepare-pg_dump-internals-for-additional-compres.patch)
download | inline diff:
From 3f319d62fb98f1bb6c60eef48b8fd04a1adca289 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 19 Dec 2022 15:02:17 +0000
Subject: [PATCH v18 1/3] Prepare pg_dump internals for additional compression
methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about it and is using it throughout.
---
src/bin/pg_dump/compress_io.c | 398 +++++++++++++++++++--------
src/bin/pg_dump/pg_backup_archiver.c | 128 +++------
src/bin/pg_dump/pg_backup_archiver.h | 27 +-
3 files changed, 324 insertions(+), 229 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index a7df600cc0..bbac154669 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -101,7 +105,7 @@ AllocateCompressor(const pg_compress_specification compression_spec,
#ifndef HAVE_LIBZ
if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
@@ -128,15 +132,25 @@ ReadDataFromArchive(ArchiveHandle *AH,
const pg_compress_specification compression_spec,
ReadFunc readF)
{
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ switch (compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
}
@@ -149,20 +163,22 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ WriteDataToArchiveNone(AH, cs, data, dLen);
+ break;
case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
}
}
@@ -173,10 +189,26 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
free(cs);
}
@@ -391,10 +423,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_specification compression_spec;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -482,7 +512,7 @@ cfopen_write(const char *path, const char *mode,
fp = cfopen(fname, mode, compression_spec);
free_keep_errno(fname);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("this build does not support compression with %s", "gzip");
fp = NULL; /* keep compiler quiet */
#endif
}
@@ -490,127 +520,203 @@ cfopen_write(const char *path, const char *mode,
}
/*
- * Opens file 'path' in 'mode'. If compression is GZIP, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ fp->compression_spec = compression_spec;
+
+ switch (compression_spec.algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compression_spec.level);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(path, -1, mode, compression_spec);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(NULL, fd, mode, compression_spec);
+}
int
cfread(void *ptr, int size, cfp *fp)
{
- int ret;
+ int ret = 0;
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, (FILE *) fp->fp);
+ if (ret != size && !feof((FILE *) fp->fp))
+ READ_ERROR_EXIT((FILE *) fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread((gzFile) fp->fp, ptr, size);
+ if (ret != size && !gzeof((gzFile) fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, (FILE *) fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite((gzFile) fp->fp, ptr, size);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
int
cfgetc(cfp *fp)
{
- int ret;
+ int ret = 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc((FILE *) fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT((FILE *) fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof((gzFile) fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
return ret;
@@ -619,65 +725,119 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret = NULL;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, (FILE *) fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets((gzFile) fp->fp, buf, len);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret = 0;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compression_spec.algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose((FILE *) fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose((gzFile) fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof((FILE *) fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof((gzFile) fp->fp);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7f7a0f1ce7..fb94317ad9 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -101,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -277,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -362,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -391,17 +383,24 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -1128,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
@@ -1503,58 +1502,32 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression_spec.level);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compression_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compression_spec);
if (!AH->OF)
{
@@ -1565,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename,
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1715,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2219,6 +2178,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2272,8 +2232,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index f72446ed5b..4725e49747 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
--
2.34.1
[text/x-patch] v18-0002-Introduce-Compressor-API-in-pg_dump.patch (65.1K, ../../IZzs7Z2ny5hRK1EhTynSg3DnCo_z-UvEJN4Q6Hf0CUHWx3BcNwt-2z5vvvvA-Z8PQL-AYiL9YQPlKylATZ_Z6HnJ55YZUHgL66H9MZQFi48=@pm.me/3-v18-0002-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From b6f74632599b30ee5479b548e4a906839eb308e7 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 19 Dec 2022 15:16:45 +0000
Subject: [PATCH v18 2/3] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives now need to store the compression algorithm in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 398 +++++++++++
src/bin/pg_dump/compress_gzip.h | 22 +
src/bin/pg_dump/compress_io.c | 918 +++++++-------------------
src/bin/pg_dump/compress_io.h | 71 +-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 102 +--
src/bin/pg_dump/pg_backup_archiver.h | 5 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 94 +--
src/bin/pg_dump/t/002_pg_dump.pl | 10 +-
src/tools/pginclude/cpluspluscheck | 1 +
src/tools/pgindent/typedefs.list | 2 +
13 files changed, 846 insertions(+), 802 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..95e1d6c276
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,398 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_gzip.c
+ * Routines for archivers to write an uncompressed or compressed data
+ * stream.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_gzip.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+#include <unistd.h>
+
+#include "compress_gzip.h"
+#include "pg_backup_utils.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private_data = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ if (deflateInit(zp, cs->compression_spec.level) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ cs->compression_spec = compression_spec;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+
+ cs->private_data = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ size_t ret;
+
+ ret = gzread(gzfp, ptr, size);
+ if (ret != size && !gzeof(gzfp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gzfp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+
+ return gzwrite(gzfp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gzfp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gzfp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+
+ return gzgets(gzfp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ int save_errno;
+ int ret;
+
+ CFH->private_data = NULL;
+
+ ret = gzclose(gzfp);
+
+ save_errno = errno;
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+
+ return gzeof(gzfp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gzfp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
+{
+ gzFile gzfp;
+ char mode_compression[32];
+
+ if (CFH->compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, CFH->compression_spec.level);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gzfp = gzdopen(dup(fd), mode_compression);
+ else
+ gzfp = gzopen(path, mode_compression);
+
+ if (gzfp == NULL)
+ return 1;
+
+ CFH->private_data = gzfp;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open_func(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ CFH->open_func = Gzip_open;
+ CFH->open_write_func = Gzip_open_write;
+ CFH->read_func = Gzip_read;
+ CFH->write_func = Gzip_write;
+ CFH->gets_func = Gzip_gets;
+ CFH->getc_func = Gzip_getc;
+ CFH->close_func = Gzip_close;
+ CFH->eof_func = Gzip_eof;
+ CFH->get_error_func = Gzip_get_error;
+
+ CFH->compression_spec = compression_spec;
+
+ CFH->private_data = NULL;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "gzip");
+}
+
+void
+InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "gzip");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..6dfd0eb04d
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_gzip.h
+ * Interface to compress_io.c routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_gzip.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec);
+extern void InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index bbac154669..4fe5b262ad 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -9,42 +9,44 @@
*
* This file includes two APIs for dealing with compressed data. The first
* provides more flexibility, using callbacks to read/write data from the
- * underlying stream. The second API is a wrapper around fopen/gzopen and
+ * underlying stream. The second API is a wrapper around fopen and
* friends, providing an interface similar to those, but abstracts away
- * the possible compression. Both APIs use libz for the compression, but
- * the second API uses gzip headers, so the resulting files can be easily
- * manipulated with the gzip utility.
+ * the possible compression. The second API is aimed for the resulting
+ * files can be easily manipulated with an external compression utility
+ * program.
*
* Compressor API
* --------------
*
* The interface for writing to an archive consists of three functions:
- * AllocateCompressor, WriteDataToArchive and EndCompressor. First you call
- * AllocateCompressor, then write all the data by calling WriteDataToArchive
- * as many times as needed, and finally EndCompressor. WriteDataToArchive
- * and EndCompressor will call the WriteFunc that was provided to
- * AllocateCompressor for each chunk of compressed data.
+ * AllocateCompressor, writeData, and EndCompressor. First you call
+ * AllocateCompressor, then write all the data by calling writeData as many
+ * times as needed, and finally EndCompressor. writeData will call the
+ * WriteFunc that was provided to AllocateCompressor for each chunk of
+ * compressed data.
*
- * The interface for reading an archive consists of just one function:
- * ReadDataFromArchive. ReadDataFromArchive reads the whole compressed input
- * stream, by repeatedly calling the given ReadFunc. ReadFunc returns the
- * compressed data chunk at a time, and ReadDataFromArchive decompresses it
- * and passes the decompressed data to ahwrite(), until ReadFunc returns 0
- * to signal EOF.
- *
- * The interface is the same for compressed and uncompressed streams.
+ * The interface for reading an archive consists of the same three functions:
+ * AllocateCompressor, readData, and EndCompressor. First you call
+ * AllocateCompressor, then read all the data by calling readData to read the
+ * whole compressed stream which repeatedly calls the given ReadFunc. ReadFunc
+ * returns the compressed data chunk at a time, and readData decompresses it
+ * and passes the decompressed data to ahwrite(), until ReadFunc returns 0 to
+ * signal EOF. The interface is the same for compressed and uncompressed
+ * streams.
*
* Compressed stream API
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -53,7 +55,11 @@
*/
#include "postgres_fe.h"
+#include <sys/stat.h>
+#include <unistd.h>
+
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,85 +71,70 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+/* Private routines that support uncompressed data I/O */
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_specification compression_spec;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
+
+ free(buf);
+}
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs,
+ const pg_compress_specification compression_spec)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+
+ cs->compression_spec = compression_spec;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compression_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compression_spec = compression_spec;
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, cs->compression_spec.level);
-#endif
-
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF)
-{
switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs, compression_spec);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
+ InitCompressorGzip(cs, compression_spec);
break;
case PG_COMPRESSION_LZ4:
pg_fatal("compression with %s is not yet supported", "LZ4");
@@ -152,35 +143,8 @@ ReadDataFromArchive(ArchiveHandle *AH,
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ return cs;
}
/*
@@ -189,402 +153,178 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- free(cs);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
+/*----------------------
+ * Compressed stream API
+ *----------------------
*/
-static void
-InitCompressorZlib(CompressorState *cs, int level)
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
+
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
}
+/* free() without changing errno; useful in several places below */
static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
+free_keep_errno(void *p)
{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
+ int save_errno = errno;
- free(cs->zlibOut);
- free(cs->zp);
+ free(p);
+ errno = save_errno;
}
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
+/*
+ * Compression None implementation
+ */
+static size_t
+read_none(void *ptr, size_t size, CompressFileHandle *CFH)
{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
+ FILE *fp = (FILE *) CFH->private_data;
+ size_t ret;
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
+ if (size == 0)
+ return 0;
- if (res == Z_STREAM_END)
- break;
- }
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
+
+ return ret;
}
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+static size_t
+write_none(const void *ptr, size_t size, CompressFileHandle *CFH)
{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
+ return fwrite(ptr, 1, size, (FILE *) CFH->private_data);
}
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
+static const char *
+get_error_none(CompressFileHandle *CFH)
{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
+ return strerror(errno);
+}
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
+static char *
+gets_none(char *ptr, int size, CompressFileHandle *CFH)
+{
+ return fgets(ptr, size, (FILE *) CFH->private_data);
+}
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
+static int
+getc_none(CompressFileHandle *CFH)
+{
+ FILE *fp = (FILE *) CFH->private_data;
+ int ret;
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
+ return ret;
}
-#endif /* HAVE_LIBZ */
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
+static int
+close_none(CompressFileHandle *CFH)
{
- size_t cnt;
- char *buf;
- size_t buflen;
+ FILE *fp = (FILE *) CFH->private_data;
+ int ret = 0;
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
+ CFH->private_data = NULL;
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
+ if (fp)
+ ret = fclose(fp);
- free(buf);
+ return ret;
}
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+
+static int
+eof_none(CompressFileHandle *CFH)
{
- cs->writeF(AH, data, dLen);
+ return feof((FILE *) CFH->private_data);
}
-
-/*----------------------
- * Compressed stream API
- *----------------------
- */
-
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+open_none(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
{
- pg_compress_specification compression_spec;
- void *fp;
-};
+ Assert(CFH->private_data == NULL);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (fd >= 0)
+ CFH->private_data = fdopen(dup(fd), mode);
+ else
+ CFH->private_data = fopen(path, mode);
-/* free() without changing errno; useful in several places below */
-static void
-free_keep_errno(void *p)
-{
- int save_errno = errno;
+ if (CFH->private_data == NULL)
+ return 1;
- free(p);
- errno = save_errno;
+ return 0;
}
-/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_read(const char *path, const char *mode)
+static int
+open_write_none(const char *path, const char *mode, CompressFileHandle *CFH)
{
- cfp *fp;
-
- pg_compress_specification compression_spec = {0};
+ Assert(CFH->private_data == NULL);
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- {
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(path, mode, compression_spec);
- }
- else
-#endif
- {
- compression_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compression_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
+ CFH->private_data = fopen(path, mode);
+ if (CFH->private_data == NULL)
+ return 1;
- fname = psprintf("%s.gz", path);
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
+ return 0;
}
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static void
+InitCompressNone(CompressFileHandle *CFH,
+ const pg_compress_specification compression_spec)
{
- cfp *fp;
-
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compression_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ CFH->open_func = open_none;
+ CFH->open_write_func = open_write_none;
+ CFH->read_func = read_none;
+ CFH->write_func = write_none;
+ CFH->gets_func = gets_none;
+ CFH->getc_func = getc_none;
+ CFH->close_func = close_none;
+ CFH->eof_func = eof_none;
+ CFH->get_error_func = get_error_none;
+
+ CFH->private_data = NULL;
}
/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
+ * Public interface
*/
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_specification compression_spec)
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compression_spec)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ CompressFileHandle *CFH;
- fp->compression_spec = compression_spec;
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
+ InitCompressNone(CFH, compression_spec);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
+ InitCompressGzip(CFH, compression_spec);
break;
case PG_COMPRESSION_LZ4:
pg_fatal("compression with %s is not yet supported", "LZ4");
@@ -594,266 +334,88 @@ cfopen_internal(const char *path, int fd, const char *mode,
break;
}
- return fp;
+ return CFH;
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
+ *
+ * On failure, return NULL with an error code in errno.
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- return cfopen_internal(path, -1, mode, compression_spec);
-}
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compression_spec = {0};
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compression_spec)
-{
- return cfopen_internal(NULL, fd, mode, compression_spec);
-}
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
-int
-cfread(void *ptr, int size, cfp *fp)
-{
- int ret = 0;
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- if (size == 0)
- return 0;
+ fname = strdup(path);
- switch (fp->compression_spec.algorithm)
+ if (hasSuffix(fname, ".gz"))
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, (FILE *) fp->fp);
- if (ret != size && !feof((FILE *) fp->fp))
- READ_ERROR_EXIT((FILE *) fp->fp);
+ bool exists;
- break;
- case PG_COMPRESSION_GZIP:
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- ret = gzread((gzFile) fp->fp, ptr, size);
- if (ret != size && !gzeof((gzFile) fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-int
-cfwrite(const void *ptr, int size, cfp *fp)
-{
- int ret = 0;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, (FILE *) fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite((gzFile) fp->fp, ptr, size);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-int
-cfgetc(cfp *fp)
-{
- int ret = 0;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc((FILE *) fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT((FILE *) fp->fp);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof((gzFile) fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-char *
-cfgets(cfp *fp, char *buf, int len)
-{
- char *ret = NULL;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, (FILE *) fp->fp);
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets((gzFile) fp->fp, buf, len);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-int
-cfclose(cfp *fp)
-{
- int ret = 0;
-
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
}
- switch (fp->compression_spec.algorithm)
+ CFH = InitCompressFileHandle(compression_spec);
+ if (CFH->open_func(fname, -1, mode, CFH))
{
- case PG_COMPRESSION_NONE:
- ret = fclose((FILE *) fp->fp);
- fp->fp = NULL;
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose((gzFile) fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
+ free_keep_errno(CFH);
+ CFH = NULL;
}
+ free_keep_errno(fname);
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
int
-cfeof(cfp *fp)
+DestroyCompressFileHandle(CompressFileHandle *CFH)
{
int ret = 0;
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof((FILE *) fp->fp);
+ if (CFH->private_data)
+ ret = CFH->close_func(CFH);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof((gzFile) fp->fp);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ free_keep_errno(CFH);
return ret;
}
-
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
-#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
-
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
- }
-
- return strerror(errno);
-}
-
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
-{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
-
- if (filenamelen < suffixlen)
- return 0;
-
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
-
-#endif
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index 6fad6c2cd5..62e3da1b1d 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,63 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ pg_compress_specification compression_spec;
+ void *private_data;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open_func) (const char *path, int fd, const char *mode,
+ CompressFileHandle *CFH);
+ int (*open_write_func) (const char *path, const char *mode,
+ CompressFileHandle *cxt);
+ size_t (*read_func) (void *ptr, size_t size, CompressFileHandle *CFH);
+ size_t (*write_func) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets_func) (char *s, int size, CompressFileHandle *CFH);
+ int (*getc_func) (CompressFileHandle *CFH);
+ int (*eof_func) (CompressFileHandle *CFH);
+ int (*close_func) (CompressFileHandle *CFH);
+ const char *(*get_error_func) (CompressFileHandle *CFH);
+
+ pg_compress_specification compression_spec;
+ void *private_data;
+};
-typedef struct cfp cfp;
+extern CompressFileHandle *InitCompressFileHandle(
+ const pg_compress_specification compression_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compression_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle *InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle *CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index d96e566846..0c73a4707e 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,5 +1,6 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index fb94317ad9..1f207c6f4d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1127,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1143,9 +1143,10 @@ PrintTOCSummary(Archive *AHX)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression_spec.level);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1502,6 +1503,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1526,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compression_spec);
- else
- AH->OF = cfopen(filename, mode, compression_spec);
+ CFH = InitCompressFileHandle(compression_spec);
- if (!AH->OF)
+ if (CFH->open_func(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1690,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write_func(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2036,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2078,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2181,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2237,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open_func(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,12 +3653,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- /*
- * For now the compression type is implied by the level. This will need
- * to change once support for more compression algorithms is added,
- * requiring a format bump.
- */
- WriteInt(AH, AH->compression_spec.level);
+ AH->WriteBytePtr(AH, AH->compression_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3722,10 +3724,11 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
- /* Guess the compression method based on the level */
- AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compression_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
+ /* Guess the compression method based on the level */
if (AH->version < K_VERS_1_4)
AH->compression_spec.level = AH->ReadBytePtr(AH);
else
@@ -3737,10 +3740,17 @@ ReadHead(ArchiveHandle *AH)
else
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("archive is compressed, but this installation does not support compression");
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
+ if (unsupported)
+ pg_fatal("archive is compressed, but this installation does not support compression");
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 4725e49747..18b38c17ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,13 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add
+ * compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index d1e54644a9..512ab043af 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compression_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index f6aee775eb..b6d025576f 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,8 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
-
- cfp *LOsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *dataFH; /* currently open data file */
+ CompressFileHandle *LOsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +197,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +217,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +326,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compression_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+
+ if (ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +345,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write_func(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error_func(CFH));
}
}
@@ -370,7 +370,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +385,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read_func(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +434,7 @@ _LoadLOs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +442,14 @@ _LoadLOs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->LOsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->LOsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the LOs TOC file line-by-line, and process each LO */
- while ((cfgets(ctx->LOsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
{
char lofname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +464,11 @@ _LoadLOs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreLO(AH, oid);
}
- if (!cfeof(ctx->LOsTocFH))
+ if (!CFH->eof_func(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->LOsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +488,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write_func(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error_func(CFH));
}
return 1;
@@ -512,8 +513,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc_func(CFH);
}
/*
@@ -524,15 +526,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write_func(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error_func(CFH));
}
}
@@ -545,12 +548,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read_func(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +577,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +588,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compression_spec);
+ if (tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +602,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +653,8 @@ _StartLOs(ArchiveHandle *AH, TocEntry *te)
/* The LO TOC file is never compressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->LOsTocFH = cfopen_write(fname, "ab", compression_spec);
- if (ctx->LOsTocFH == NULL)
+ ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
+ if (ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +671,8 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+ if (ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,18 +685,19 @@ static void
_EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->LOsTocFH;
char buf[50];
int len;
- /* Close the LO data file itself */
- if (cfclose(ctx->dataFH) != 0)
- pg_fatal("could not close LO data file: %m");
+ /* Close the BLOB data file itself */
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
+ pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the LO in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->LOsTocFH) != len)
- pg_fatal("could not write to LOs TOC file");
+ if (CFH->write_func(buf, len, CFH) != len)
+ pg_fatal("could not write to blobs TOC file");
}
/*
@@ -706,8 +710,8 @@ _EndLOs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->LOsTocFH) != 0)
- pg_fatal("could not close LOs TOC file: %m");
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
+ pg_fatal("could not close blobs TOC file: %m");
ctx->LOsTocFH = NULL;
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 1c7fc728c2..39daa1fc43 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -94,7 +94,7 @@ my %pgdump_runs = (
command => [
'pg_restore', '-l', "$tempdir/compression_gzip_custom.dump",
],
- expected => qr/Compression: 1/,
+ expected => qr/Compression: gzip/,
name => 'data content is gzip-compressed'
},
},
@@ -239,8 +239,8 @@ my %pgdump_runs = (
command =>
[ 'pg_restore', '-l', "$tempdir/defaults_custom_format.dump", ],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip-compressed by default if available',
},
},
@@ -264,8 +264,8 @@ my %pgdump_runs = (
command =>
[ 'pg_restore', '-l', "$tempdir/defaults_dir_format", ],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip-compressed by default',
},
glob_patterns => [
diff --git a/src/tools/pginclude/cpluspluscheck b/src/tools/pginclude/cpluspluscheck
index b393f2a2ea..8805237edb 100755
--- a/src/tools/pginclude/cpluspluscheck
+++ b/src/tools/pginclude/cpluspluscheck
@@ -150,6 +150,7 @@ do
# pg_dump is not C++-clean because it uses "public" and "namespace"
# as field names, which is unfortunate but we won't change it now.
+ test "$f" = src/bin/pg_dump/compress_gzip.h && continue
test "$f" = src/bin/pg_dump/compress_io.h && continue
test "$f" = src/bin/pg_dump/parallel.h && continue
test "$f" = src/bin/pg_dump/pg_backup_archiver.h && continue
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 60c71d05fe..81a451641a 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -428,6 +428,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
+CompressFileHandle
CompressionLocation
CompressorState
ComputeXidHorizonsResult
@@ -1034,6 +1035,7 @@ GucStack
GucStackState
GucStringAssignHook
GucStringCheckHook
+GzipCompressorState
HANDLE
HASHACTION
HASHBUCKET
--
2.34.1
[text/x-patch] v18-0003-Add-LZ4-compression-in-pg_-dump-restore.patch (28.8K, ../../IZzs7Z2ny5hRK1EhTynSg3DnCo_z-UvEJN4Q6Hf0CUHWx3BcNwt-2z5vvvvA-Z8PQL-AYiL9YQPlKylATZ_Z6HnJ55YZUHgL66H9MZQFi48=@pm.me/4-v18-0003-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From 03c101e84b74078b3bb4a7b85637db71157fd2be Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Mon, 19 Dec 2022 15:43:56 +0000
Subject: [PATCH v18 3/3] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 13 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 11 +-
src/bin/pg_dump/compress_lz4.c | 618 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 22 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 5 +-
src/bin/pg_dump/t/002_pg_dump.pl | 82 +++-
src/tools/pginclude/cpluspluscheck | 1 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 756 insertions(+), 21 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2c938cd7e1..49d218905f 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -330,9 +330,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -654,7 +655,7 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression.
+ <literal>lz4</literal> or <literal>none</literal> for no compression.
A compression detail string can optionally be specified. If the
detail string is an integer, it specifies the compression level.
Otherwise, it should be a comma-separated list of items, each of the
@@ -675,8 +676,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 4fe5b262ad..fa971fa95c 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -60,6 +60,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -137,7 +138,7 @@ AllocateCompressor(const pg_compress_specification compression_spec,
InitCompressorGzip(cs, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
+ InitCompressorLZ4(cs, compression_spec);
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
@@ -327,7 +328,7 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
InitCompressGzip(CFH, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
+ InitCompressLZ4(CFH, compression_spec);
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
@@ -341,12 +342,12 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
- * order.
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
*
* On failure, return NULL with an error code in errno.
+ *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..c97e16187a
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,618 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_lz4.c
+ * Routines for archivers to write an uncompressed or compressed data
+ * stream.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_lz4.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*
+ * LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
+ * Redefine it for installations with a lesser version.
+ */
+#ifndef LZ4F_HEADER_SIZE_MAX
+#define LZ4F_HEADER_SIZE_MAX 32
+#endif
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private_data;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private_data;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private_data = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ cs->compression_spec = compression_spec;
+
+ /* Will be lazy init'd */
+ cs->private_data = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle *CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle *CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private_data;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open_func(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ LZ4File *lz4fp;
+
+ CFH->open_func = LZ4File_open;
+ CFH->open_write_func = LZ4File_open_write;
+ CFH->read_func = LZ4File_read;
+ CFH->write_func = LZ4File_write;
+ CFH->gets_func = LZ4File_gets;
+ CFH->getc_func = LZ4File_getc;
+ CFH->eof_func = LZ4File_eof;
+ CFH->close_func = LZ4File_close;
+ CFH->get_error_func = LZ4File_get_error;
+
+ CFH->compression_spec = compression_spec;
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (CFH->compression_spec.level >= 0)
+ lz4fp->prefs.compressionLevel = CFH->compression_spec.level;
+
+ CFH->private_data = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "LZ4");
+}
+
+void
+InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "LZ4");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..74595db1b9
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_lz4.h
+ * Interface to compress_io.c routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_lz4.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec);
+extern void InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 0c73a4707e..b27e92ffd0 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -1,6 +1,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -15,7 +16,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -83,7 +84,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f207c6f4d..119b7f2553 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -395,6 +395,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2074,7 +2078,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2084,6 +2088,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3747,6 +3755,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
pg_fatal("archive is compressed, but this installation does not support compression");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 44d957c038..1bb874b8e3 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -715,13 +715,12 @@ main(int argc, char **argv)
case PG_COMPRESSION_NONE:
/* fallthrough */
case PG_COMPRESSION_GZIP:
+ /* fallthrough */
+ case PG_COMPRESSION_LZ4:
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
}
/*
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 39daa1fc43..d3f28dfba7 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -139,6 +139,80 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=lz4', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ command_like => {
+ command => [
+ 'pg_restore',
+ '-l', "$tempdir/compression_lz4_custom.dump",
+ ],
+ expected => qr/Compression: lz4/,
+ name => 'data content is lz4 compressed'
+ },
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ # Verify that data files where compressed
+ glob_patterns => [
+ "$tempdir/compression_lz4_dir/toc.dat",
+ "$tempdir/compression_lz4_dir/*.dat.lz4",
+ ],
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4175,11 +4249,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
diff --git a/src/tools/pginclude/cpluspluscheck b/src/tools/pginclude/cpluspluscheck
index 8805237edb..d44ce64fd7 100755
--- a/src/tools/pginclude/cpluspluscheck
+++ b/src/tools/pginclude/cpluspluscheck
@@ -152,6 +152,7 @@ do
# as field names, which is unfortunate but we won't change it now.
test "$f" = src/bin/pg_dump/compress_gzip.h && continue
test "$f" = src/bin/pg_dump/compress_io.h && continue
+ test "$f" = src/bin/pg_dump/compress_lz4.h && continue
test "$f" = src/bin/pg_dump/parallel.h && continue
test "$f" = src/bin/pg_dump/pg_backup_archiver.h && continue
test "$f" = src/bin/pg_dump/pg_dump.h && continue
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 81a451641a..0e25c7f58a 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1381,6 +1381,7 @@ LWLock
LWLockHandle
LWLockMode
LWLockPadded
+LZ4CompressorState
LZ4F_compressionContext_t
LZ4F_decompressOptions_t
LZ4F_decompressionContext_t
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-19 17:27 ` Justin Pryzby <[email protected]>
2022-12-20 11:19 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-12-19 17:27 UTC (permalink / raw)
To: [email protected]; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Mon, Dec 19, 2022 at 05:03:21PM +0000, [email protected] wrote:
> > > 001 still doesn't compile on freebsd, and 002 doesn't compile on
> > > windows. Have you checked test results from cirrusci on your private
> > > github account ?
>
> There are still known gaps in 0002 and 0003, for example documentation,
> and I have not been focusing too much on those. You are right, it is helpful
> and kind to try to reduce the noise. The attached should have hopefully
> tackled the ci errors.
Yep. Are you using cirrusci under your github account ?
> > FYI, I have re-added an entry to the CF app to get some automated
> > coverage:
> > https://commitfest.postgresql.org/41/3571/
>
> Much obliged. Should I change the state to "ready for review" when post a
> new version or should I leave that to the senior personnel?
It's better to update it to reflect what you think its current status
is. If you think it's ready for review.
> > > 002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
> > > doesn't store the passed-in compression_spec.
>
> I am afraid I have not been able to reproduce this error. I tried both
> debian and freebsd after I addressed the compilation warnings. Which
> error did you get? Is it still present in the attached?
It's not that there's an error - it's that compression isn't working.
$ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fp regression |wc -c
659956
$ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fp regression |wc -c
637192
$ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fc regression |wc -c
1954890
$ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fc regression |wc -c
1954890
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-19 17:27 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-12-20 11:19 ` [email protected]
2022-12-20 15:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-20 11:19 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Monday, December 19th, 2022 at 6:27 PM, Justin Pryzby <[email protected]> wrote:
>
>
> On Mon, Dec 19, 2022 at 05:03:21PM +0000, [email protected] wrote:
>
> > > > 001 still doesn't compile on freebsd, and 002 doesn't compile on
> > > > windows. Have you checked test results from cirrusci on your private
> > > > github account ?
> >
> > There are still known gaps in 0002 and 0003, for example documentation,
> > and I have not been focusing too much on those. You are right, it is helpful
> > and kind to try to reduce the noise. The attached should have hopefully
> > tackled the ci errors.
>
>
> Yep. Are you using cirrusci under your github account ?
Thank you. To be very honest, I am not using github exclusively to post patches.
Sometimes I do, sometimes I do not. Is github a requirement?
To answer your question, some of my github accounts are integrated with cirrusci,
others are not.
The current cfbot build is green for what is worth.
https://cirrus-ci.com/build/5934319840002048
>
> > > FYI, I have re-added an entry to the CF app to get some automated
> > > coverage:
> > > https://commitfest.postgresql.org/41/3571/
> >
> > Much obliged. Should I change the state to "ready for review" when post a
> > new version or should I leave that to the senior personnel?
>
>
> It's better to update it to reflect what you think its current status
> is. If you think it's ready for review.
Thank you.
>
> > > > 002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
> > > > doesn't store the passed-in compression_spec.
> >
> > I am afraid I have not been able to reproduce this error. I tried both
> > debian and freebsd after I addressed the compilation warnings. Which
> > error did you get? Is it still present in the attached?
>
>
> It's not that there's an error - it's that compression isn't working.
>
> $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fp regression |wc -c
> 659956
> $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fp regression |wc -c
> 637192
>
> $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fc regression |wc -c
> 1954890
> $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fc regression |wc -c
> 1954890
>
Thank you. Now I understand what you mean. Trying the same on top of v18-0003
on Ubuntu 22.04 yields:
$ for compression in none gzip:1 gzip:6 gzip:9; do \
pg_dump --format=custom --compress="$compression" -f regression."$compression".dump -d regression; \
wc -c regression."$compression".dump; \
done;
14963753 regression.none.dump
3600183 regression.gzip:1.dump
3223755 regression.gzip:6.dump
3196903 regression.gzip:9.dump
and on FreeBSD 13.1
$ for compression in none gzip:1 gzip:6 gzip:9; do \
pg_dump --format=custom --compress="$compression" -f regression."$compression".dump -d regression; \
wc -c regression."$compression".dump; \
done;
14828822 regression.none.dump
3584304 regression.gzip:1.dump
3208548 regression.gzip:6.dump
3182044 regression.gzip:9.dump
Although there are some variations between the installations, within the same
installation the size of the dump file is shrinking as expected.
Investigating a bit further on the issue, you are correct in identifying an
issue in v17. Up until v16, the compressor function looked like:
+InitCompressorGzip(CompressorState *cs, int compressionLevel)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+ gzipcs->compressionLevel = compressionLevel;
V17 considered that more options could become available in the future
and changed the signature of the relevant Init functions to:
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+
V18 reinstated the assignment in similar fashion to InitCompressorNone and
InitCompressorLz4:
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ cs->compression_spec = compression_spec;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
A test case can be added which performs a check similar to the loop above.
Create a custom dump with the least and most compression for each method.
Then verify that the output sizes differ as expected. This addition could
become 0001 in the current series.
Thoughts?
Cheers,
//Georgios
> --
> Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-19 17:27 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-20 11:19 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-20 15:26 ` Justin Pryzby <[email protected]>
2022-12-21 10:09 ` Re: Add LZ4 compression in pg_dump [email protected]
0 siblings, 1 reply; 67+ messages in thread
From: Justin Pryzby @ 2022-12-20 15:26 UTC (permalink / raw)
To: [email protected]; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Tue, Dec 20, 2022 at 11:19:15AM +0000, [email protected] wrote:
> ------- Original Message -------
> On Monday, December 19th, 2022 at 6:27 PM, Justin Pryzby <[email protected]> wrote:
> > On Mon, Dec 19, 2022 at 05:03:21PM +0000, [email protected] wrote:
> >
> > > > > 001 still doesn't compile on freebsd, and 002 doesn't compile on
> > > > > windows. Have you checked test results from cirrusci on your private
> > > > > github account ?
> > >
> > > There are still known gaps in 0002 and 0003, for example documentation,
> > > and I have not been focusing too much on those. You are right, it is helpful
> > > and kind to try to reduce the noise. The attached should have hopefully
> > > tackled the ci errors.
> >
> >
> > Yep. Are you using cirrusci under your github account ?
>
> Thank you. To be very honest, I am not using github exclusively to post patches.
> Sometimes I do, sometimes I do not. Is github a requirement?
Github isn't a requirement for postgres (but cirrusci only supports
github). I wasn't not trying to say that it's required, only trying to
make sure that you (and others) know that it's available, since our
cirrus.yml is relatively new.
> > > > > 002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
> > > > > doesn't store the passed-in compression_spec.
> > >
> > > I am afraid I have not been able to reproduce this error. I tried both
> > > debian and freebsd after I addressed the compilation warnings. Which
> > > error did you get? Is it still present in the attached?
> >
> >
> > It's not that there's an error - it's that compression isn't working.
> >
> > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fp regression |wc -c
> > 659956
> > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fp regression |wc -c
> > 637192
> >
> > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fc regression |wc -c
> > 1954890
> > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fc regression |wc -c
> > 1954890
> >
>
> Thank you. Now I understand what you mean. Trying the same on top of v18-0003
> on Ubuntu 22.04 yields:
You're right; this seems to be fixed in v18. Thanks.
It looks like I'd forgotten to run "meson test tmp_install", so had
retested v17...
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-19 17:27 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-20 11:19 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-20 15:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
@ 2022-12-21 10:09 ` [email protected]
2022-12-22 17:08 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
0 siblings, 1 reply; 67+ messages in thread
From: [email protected] @ 2022-12-21 10:09 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
------- Original Message -------
On Tuesday, December 20th, 2022 at 4:26 PM, Justin Pryzby <[email protected]> wrote:
>
>
> On Tue, Dec 20, 2022 at 11:19:15AM +0000, [email protected] wrote:
>
> > ------- Original Message -------
> > On Monday, December 19th, 2022 at 6:27 PM, Justin Pryzby [email protected] wrote:
> >
> > > On Mon, Dec 19, 2022 at 05:03:21PM +0000, [email protected] wrote:
> > >
> > > > > > 001 still doesn't compile on freebsd, and 002 doesn't compile on
> > > > > > windows. Have you checked test results from cirrusci on your private
> > > > > > github account ?
> > > >
> > > > There are still known gaps in 0002 and 0003, for example documentation,
> > > > and I have not been focusing too much on those. You are right, it is helpful
> > > > and kind to try to reduce the noise. The attached should have hopefully
> > > > tackled the ci errors.
> > >
> > > Yep. Are you using cirrusci under your github account ?
> >
> > Thank you. To be very honest, I am not using github exclusively to post patches.
> > Sometimes I do, sometimes I do not. Is github a requirement?
>
>
> Github isn't a requirement for postgres (but cirrusci only supports
> github). I wasn't not trying to say that it's required, only trying to
> make sure that you (and others) know that it's available, since our
> cirrus.yml is relatively new.
Got it. Thank you very much for spreading the word. It is a useful feature which
should be known.
>
> > > > > > 002 breaks "pg_dump -Fc -Z2" because (I think) AllocateCompressor()
> > > > > > doesn't store the passed-in compression_spec.
> > > >
> > > > I am afraid I have not been able to reproduce this error. I tried both
> > > > debian and freebsd after I addressed the compilation warnings. Which
> > > > error did you get? Is it still present in the attached?
> > >
> > > It's not that there's an error - it's that compression isn't working.
> > >
> > > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fp regression |wc -c
> > > 659956
> > > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fp regression |wc -c
> > > 637192
> > >
> > > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z1 -Fc regression |wc -c
> > > 1954890
> > > $ ./tmp_install/usr/local/pgsql/bin/pg_dump -h /tmp -Z2 -Fc regression |wc -c
> > > 1954890
> >
> > Thank you. Now I understand what you mean. Trying the same on top of v18-0003
> > on Ubuntu 22.04 yields:
>
>
> You're right; this seems to be fixed in v18. Thanks.
Great. Still there was a bug in v17 which you discovered. Thank you for the review
effort.
Please find in the attached v19 an extra check right before calling deflateInit().
This check will verify that only compressed output will be generated for this
method.
Also v19 is rebased on top f450695e889 and applies cleanly.
Cheers.
//Georgios
> --
> Justin
Attachments:
[text/x-patch] v19-0001-Prepare-pg_dump-internals-for-additional-compres.patch (21.9K, ../../f8Wmj1EmRT7CHBfMcg0jM_Wts_hyV22WsLWzf5X5KImQo_5CtZXHK3UjxL8Wne6g7zSccwXLNToLCINfEIPbNni53gKO17AuHplMjm-XBmA=@pm.me/2-v19-0001-Prepare-pg_dump-internals-for-additional-compres.patch)
download | inline diff:
From 6e3c760dfef68e805b26fba1b499b9f5ef46d86b Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 21 Dec 2022 09:49:23 +0000
Subject: [PATCH v19 1/3] Prepare pg_dump internals for additional compression
methods.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about it and is using it throughout.
---
src/bin/pg_dump/compress_io.c | 389 +++++++++++++++++++--------
src/bin/pg_dump/pg_backup_archiver.c | 128 +++------
src/bin/pg_dump/pg_backup_archiver.h | 27 +-
3 files changed, 318 insertions(+), 226 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 26967eb618..4bad69f4bd 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -128,15 +132,24 @@ ReadDataFromArchive(ArchiveHandle *AH,
const pg_compress_specification compression_spec,
ReadFunc readF)
{
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ switch (compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("this build does not support compression with %s", "gzip");
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
}
@@ -149,6 +162,9 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->compression_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ WriteDataToArchiveNone(AH, cs, data, dLen);
+ break;
case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
@@ -156,13 +172,11 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
pg_fatal("this build does not support compression with %s", "gzip");
#endif
break;
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
case PG_COMPRESSION_LZ4:
- /* fallthrough */
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
case PG_COMPRESSION_ZSTD:
- pg_fatal("invalid compression method");
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
}
}
@@ -173,10 +187,26 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
free(cs);
}
@@ -391,10 +421,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_specification compression_spec;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -490,127 +518,202 @@ cfopen_write(const char *path, const char *mode,
}
/*
- * Opens file 'path' in 'mode'. If compression is GZIP, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_specification compression_spec)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ fp->compression_spec = compression_spec;
+
+ switch (compression_spec.algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compression_spec.level);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(path, -1, mode, compression_spec);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compression_spec)
+{
+ return cfopen_internal(NULL, fd, mode, compression_spec);
+}
int
cfread(void *ptr, int size, cfp *fp)
{
- int ret;
+ int ret = 0;
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, (FILE *) fp->fp);
+ if (ret != size && !feof((FILE *) fp->fp))
+ READ_ERROR_EXIT((FILE *) fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread((gzFile) fp->fp, ptr, size);
+ if (ret != size && !gzeof((gzFile) fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, (FILE *) fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite((gzFile) fp->fp, ptr, size);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
int
cfgetc(cfp *fp)
{
- int ret;
+ int ret = 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compression_spec.algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc((FILE *) fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT((FILE *) fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof((gzFile) fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
return ret;
@@ -619,65 +722,119 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret = NULL;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, (FILE *) fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets((gzFile) fp->fp, buf, len);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret = 0;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compression_spec.algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose((FILE *) fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose((gzFile) fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compression_spec.algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof((FILE *) fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof((gzFile) fp->fp);
+#else
+ pg_fatal("this build does not support compression with %s",
+ "gzip");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ case PG_COMPRESSION_LZ4:
+ pg_fatal("compression with %s is not yet supported", "LZ4");
+ break;
+ case PG_COMPRESSION_ZSTD:
+ pg_fatal("compression with %s is not yet supported", "ZSTD");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("this build does not support compression with %s", "gzip");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 7f7a0f1ce7..fb94317ad9 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -101,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -277,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -362,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -391,17 +383,24 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
+ supports_compression = true;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->compression_spec.algorithm == PG_COMPRESSION_GZIP &&
AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -1128,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
@@ -1503,58 +1502,32 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression_spec.level);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compression_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compression_spec);
if (!AH->OF)
{
@@ -1565,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename,
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1715,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2219,6 +2178,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2272,8 +2232,8 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index f72446ed5b..4725e49747 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
--
2.34.1
[text/x-patch] v19-0002-Introduce-Compressor-API-in-pg_dump.patch (65.3K, ../../f8Wmj1EmRT7CHBfMcg0jM_Wts_hyV22WsLWzf5X5KImQo_5CtZXHK3UjxL8Wne6g7zSccwXLNToLCINfEIPbNni53gKO17AuHplMjm-XBmA=@pm.me/3-v19-0002-Introduce-Compressor-API-in-pg_dump.patch)
download | inline diff:
From 5406ebc73d7e6c068b9cb10a05210ee852685b73 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 21 Dec 2022 09:49:31 +0000
Subject: [PATCH v19 2/3] Introduce Compressor API in pg_dump
The purpose of this API is to allow for easier addition of new compression
methods. CompressFileHandle is substituting the cfp* family of functions under a
struct of function pointers for opening, writing, etc. The implementor of a new
compression method is now able to "simply" just add those definitions.
Custom compressed archives now need to store the compression algorithm in their
header. This requires a bump in the version number. The level of compression
is no longer stored in the dump as it is irrelevant.
---
src/bin/pg_dump/Makefile | 1 +
src/bin/pg_dump/compress_gzip.c | 405 ++++++++++++
src/bin/pg_dump/compress_gzip.h | 22 +
src/bin/pg_dump/compress_io.c | 914 +++++++-------------------
src/bin/pg_dump/compress_io.h | 71 +-
src/bin/pg_dump/meson.build | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 102 +--
src/bin/pg_dump/pg_backup_archiver.h | 5 +-
src/bin/pg_dump/pg_backup_custom.c | 23 +-
src/bin/pg_dump/pg_backup_directory.c | 94 +--
src/bin/pg_dump/t/002_pg_dump.pl | 10 +-
src/tools/pginclude/cpluspluscheck | 1 +
src/tools/pgindent/typedefs.list | 2 +
13 files changed, 852 insertions(+), 799 deletions(-)
create mode 100644 src/bin/pg_dump/compress_gzip.c
create mode 100644 src/bin/pg_dump/compress_gzip.h
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 9dc5a784dd..29eab02d37 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -24,6 +24,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
+ compress_gzip.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c
new file mode 100644
index 0000000000..60fb95d7b7
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.c
@@ -0,0 +1,405 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_gzip.c
+ * Routines for archivers to write an uncompressed or compressed data
+ * stream.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_gzip.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+#include <unistd.h>
+
+#include "compress_gzip.h"
+#include "pg_backup_utils.h"
+
+#ifdef HAVE_LIBZ
+#include "zlib.h"
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+typedef struct GzipCompressorState
+{
+ z_streamp zp;
+
+ void *outbuf;
+ size_t outsize;
+} GzipCompressorState;
+
+/* Private routines that support gzip compressed data I/O */
+static void
+DeflateCompressorGzip(ArchiveHandle *AH, CompressorState *cs, bool flush)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data;
+ z_streamp zp = gzipcs->zp;
+ void *out = gzipcs->outbuf;
+ int res = Z_OK;
+
+ while (gzipcs->zp->avail_in != 0 || flush)
+ {
+ res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
+ if (res == Z_STREAM_ERROR)
+ pg_fatal("could not compress data: %s", zp->msg);
+ if ((flush && (zp->avail_out < gzipcs->outsize))
+ || (zp->avail_out == 0)
+ || (zp->avail_in != 0)
+ )
+ {
+ /*
+ * Extra paranoia: avoid zero-length chunks, since a zero length
+ * chunk is the EOF marker in the custom format. This should never
+ * happen but...
+ */
+ if (zp->avail_out < gzipcs->outsize)
+ {
+ /*
+ * Any write function should do its own error checking but to
+ * make sure we do a check here as well...
+ */
+ size_t len = gzipcs->outsize - zp->avail_out;
+
+ cs->writeF(AH, (char *) out, len);
+ }
+ zp->next_out = out;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ if (res == Z_STREAM_END)
+ break;
+ }
+}
+
+static void
+EndCompressorGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data;
+ z_streamp zp;
+
+ if (gzipcs->zp)
+ {
+ zp = gzipcs->zp;
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+
+ /* Flush any remaining data from zlib buffer */
+ DeflateCompressorGzip(AH, cs, true);
+
+ if (deflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression stream: %s", zp->msg);
+
+ pg_free(gzipcs->outbuf);
+ pg_free(gzipcs->zp);
+ }
+
+ pg_free(gzipcs);
+ cs->private_data = NULL;
+}
+
+static void
+WriteDataToArchiveGzip(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ GzipCompressorState *gzipcs = (GzipCompressorState *) cs->private_data;
+ z_streamp zp;
+
+ if (!gzipcs->zp)
+ {
+ zp = gzipcs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ /*
+ * outsize is the buffer size we tell zlib it can output to. We
+ * actually allocate one extra byte because some routines want to
+ * append a trailing zero byte to the zlib output.
+ */
+ gzipcs->outbuf = pg_malloc(ZLIB_OUT_SIZE + 1);
+ gzipcs->outsize = ZLIB_OUT_SIZE;
+
+ /*
+ * A level of zero simply copies the input one block at the time. This
+ * is probably not what the user wanted when calling this interface.
+ */
+ if (cs->compression_spec.level == 0)
+ pg_fatal("requested to compress the archive yet no level was specified");
+
+ if (deflateInit(zp, cs->compression_spec.level) != Z_OK)
+ pg_fatal("could not initialize compression library: %s", zp->msg);
+
+ /* Just be paranoid - maybe End is called after Start, with no Write */
+ zp->next_out = gzipcs->outbuf;
+ zp->avail_out = gzipcs->outsize;
+ }
+
+ gzipcs->zp->next_in = (void *) unconstify(void *, data);
+ gzipcs->zp->avail_in = dLen;
+ DeflateCompressorGzip(AH, cs, false);
+}
+
+static void
+ReadDataFromArchiveGzip(ArchiveHandle *AH, CompressorState *cs)
+{
+ z_streamp zp;
+ char *out;
+ int res = Z_OK;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
+
+ zp = (z_streamp) pg_malloc(sizeof(z_stream));
+ zp->zalloc = Z_NULL;
+ zp->zfree = Z_NULL;
+ zp->opaque = Z_NULL;
+
+ buf = pg_malloc(ZLIB_IN_SIZE);
+ buflen = ZLIB_IN_SIZE;
+
+ out = pg_malloc(ZLIB_OUT_SIZE + 1);
+
+ if (inflateInit(zp) != Z_OK)
+ pg_fatal("could not initialize compression library: %s",
+ zp->msg);
+
+ /* no minimal chunk size for zlib */
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ zp->next_in = (void *) buf;
+ zp->avail_in = cnt;
+
+ while (zp->avail_in > 0)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+ }
+
+ zp->next_in = NULL;
+ zp->avail_in = 0;
+ while (res != Z_STREAM_END)
+ {
+ zp->next_out = (void *) out;
+ zp->avail_out = ZLIB_OUT_SIZE;
+ res = inflate(zp, 0);
+ if (res != Z_OK && res != Z_STREAM_END)
+ pg_fatal("could not uncompress data: %s", zp->msg);
+
+ out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
+ ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ }
+
+ if (inflateEnd(zp) != Z_OK)
+ pg_fatal("could not close compression library: %s", zp->msg);
+
+ free(buf);
+ free(out);
+ free(zp);
+}
+
+/* Public routines that support gzip compressed data I/O */
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ GzipCompressorState *gzipcs;
+
+ cs->readData = ReadDataFromArchiveGzip;
+ cs->writeData = WriteDataToArchiveGzip;
+ cs->end = EndCompressorGzip;
+
+ cs->compression_spec = compression_spec;
+
+ gzipcs = (GzipCompressorState *) pg_malloc0(sizeof(GzipCompressorState));
+
+ cs->private_data = gzipcs;
+}
+
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+static size_t
+Gzip_read(void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ size_t ret;
+
+ ret = gzread(gzfp, ptr, size);
+ if (ret != size && !gzeof(gzfp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(gzfp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+
+ return ret;
+}
+
+static size_t
+Gzip_write(const void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+
+ return gzwrite(gzfp, ptr, size);
+}
+
+static int
+Gzip_getc(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ int ret;
+
+ errno = 0;
+ ret = gzgetc(gzfp);
+ if (ret == EOF)
+ {
+ if (!gzeof(gzfp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return ret;
+}
+
+static char *
+Gzip_gets(char *ptr, int size, CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+
+ return gzgets(gzfp, ptr, size);
+}
+
+static int
+Gzip_close(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ int save_errno;
+ int ret;
+
+ CFH->private_data = NULL;
+
+ ret = gzclose(gzfp);
+
+ save_errno = errno;
+ errno = save_errno;
+
+ return ret;
+}
+
+static int
+Gzip_eof(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+
+ return gzeof(gzfp);
+}
+
+static const char *
+Gzip_get_error(CompressFileHandle *CFH)
+{
+ gzFile gzfp = (gzFile) CFH->private_data;
+ const char *errmsg;
+ int errnum;
+
+ errmsg = gzerror(gzfp, &errnum);
+ if (errnum == Z_ERRNO)
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+static int
+Gzip_open(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
+{
+ gzFile gzfp;
+ char mode_compression[32];
+
+ if (CFH->compression_spec.level != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use it
+ */
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, CFH->compression_spec.level);
+ }
+ else
+ strcpy(mode_compression, mode);
+
+ if (fd >= 0)
+ gzfp = gzdopen(dup(fd), mode_compression);
+ else
+ gzfp = gzopen(path, mode_compression);
+
+ if (gzfp == NULL)
+ return 1;
+
+ CFH->private_data = gzfp;
+
+ return 0;
+}
+
+static int
+Gzip_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
+{
+ char *fname;
+ int ret;
+ int save_errno;
+
+ fname = psprintf("%s.gz", path);
+ ret = CFH->open_func(fname, -1, mode, CFH);
+
+ save_errno = errno;
+ pg_free(fname);
+ errno = save_errno;
+
+ return ret;
+}
+
+void
+InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ CFH->open_func = Gzip_open;
+ CFH->open_write_func = Gzip_open_write;
+ CFH->read_func = Gzip_read;
+ CFH->write_func = Gzip_write;
+ CFH->gets_func = Gzip_gets;
+ CFH->getc_func = Gzip_getc;
+ CFH->close_func = Gzip_close;
+ CFH->eof_func = Gzip_eof;
+ CFH->get_error_func = Gzip_get_error;
+
+ CFH->compression_spec = compression_spec;
+
+ CFH->private_data = NULL;
+}
+#else /* HAVE_LIBZ */
+void
+InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "gzip");
+}
+
+void
+InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "gzip");
+}
+#endif /* HAVE_LIBZ */
diff --git a/src/bin/pg_dump/compress_gzip.h b/src/bin/pg_dump/compress_gzip.h
new file mode 100644
index 0000000000..6dfd0eb04d
--- /dev/null
+++ b/src/bin/pg_dump/compress_gzip.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_gzip.h
+ * Interface to compress_io.c routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_gzip.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _COMPRESS_GZIP_H_
+#define _COMPRESS_GZIP_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorGzip(CompressorState *cs, const pg_compress_specification compression_spec);
+extern void InitCompressGzip(CompressFileHandle *CFH, const pg_compress_specification compression_spec);
+
+#endif /* _COMPRESS_GZIP_H_ */
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 4bad69f4bd..6975ea6920 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -9,42 +9,44 @@
*
* This file includes two APIs for dealing with compressed data. The first
* provides more flexibility, using callbacks to read/write data from the
- * underlying stream. The second API is a wrapper around fopen/gzopen and
+ * underlying stream. The second API is a wrapper around fopen and
* friends, providing an interface similar to those, but abstracts away
- * the possible compression. Both APIs use libz for the compression, but
- * the second API uses gzip headers, so the resulting files can be easily
- * manipulated with the gzip utility.
+ * the possible compression. The second API is aimed for the resulting
+ * files can be easily manipulated with an external compression utility
+ * program.
*
* Compressor API
* --------------
*
* The interface for writing to an archive consists of three functions:
- * AllocateCompressor, WriteDataToArchive and EndCompressor. First you call
- * AllocateCompressor, then write all the data by calling WriteDataToArchive
- * as many times as needed, and finally EndCompressor. WriteDataToArchive
- * and EndCompressor will call the WriteFunc that was provided to
- * AllocateCompressor for each chunk of compressed data.
+ * AllocateCompressor, writeData, and EndCompressor. First you call
+ * AllocateCompressor, then write all the data by calling writeData as many
+ * times as needed, and finally EndCompressor. writeData will call the
+ * WriteFunc that was provided to AllocateCompressor for each chunk of
+ * compressed data.
*
- * The interface for reading an archive consists of just one function:
- * ReadDataFromArchive. ReadDataFromArchive reads the whole compressed input
- * stream, by repeatedly calling the given ReadFunc. ReadFunc returns the
- * compressed data chunk at a time, and ReadDataFromArchive decompresses it
- * and passes the decompressed data to ahwrite(), until ReadFunc returns 0
- * to signal EOF.
- *
- * The interface is the same for compressed and uncompressed streams.
+ * The interface for reading an archive consists of the same three functions:
+ * AllocateCompressor, readData, and EndCompressor. First you call
+ * AllocateCompressor, then read all the data by calling readData to read the
+ * whole compressed stream which repeatedly calls the given ReadFunc. ReadFunc
+ * returns the compressed data chunk at a time, and readData decompresses it
+ * and passes the decompressed data to ahwrite(), until ReadFunc returns 0 to
+ * signal EOF. The interface is the same for compressed and uncompressed
+ * streams.
*
* Compressed stream API
* ----------------------
*
* The compressed stream API is a wrapper around the C standard fopen() and
- * libz's gzopen() APIs. It allows you to use the same functions for
- * compressed and uncompressed streams. cfopen_read() first tries to open
- * the file with given name, and if it fails, it tries to open the same
- * file with the .gz suffix. cfopen_write() opens a file for writing, an
- * extra argument specifies if the file should be compressed, and adds the
- * .gz suffix to the filename if so. This allows you to easily handle both
- * compressed and uncompressed files.
+ * libz's gzopen() APIs and custom LZ4 calls which provide similar
+ * functionality. It allows you to use the same functions for compressed and
+ * uncompressed streams. cfopen_read() first tries to open the file with given
+ * name, and if it fails, it tries to open the same file with the .gz suffix,
+ * failing that it tries to open the same file with the .lz4 suffix.
+ * cfopen_write() opens a file for writing, an extra argument specifies the
+ * method to use should the file be compressed, and adds the appropriate
+ * suffix, .gz or .lz4, to the filename if so. This allows you to easily handle
+ * both compressed and uncompressed files.
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
@@ -53,7 +55,11 @@
*/
#include "postgres_fe.h"
+#include <sys/stat.h>
+#include <unistd.h>
+
#include "compress_io.h"
+#include "compress_gzip.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -65,84 +71,69 @@
*----------------------
*/
-/* typedef appears in compress_io.h */
-struct CompressorState
+static void
+ReadDataFromArchiveNone(ArchiveHandle *AH, CompressorState *cs)
{
- pg_compress_specification compression_spec;
- WriteFunc writeF;
+ size_t cnt;
+ char *buf;
+ size_t buflen;
-#ifdef HAVE_LIBZ
- z_streamp zp;
- char *zlibOut;
- size_t zlibOutSize;
-#endif
-};
+ buf = pg_malloc(ZLIB_OUT_SIZE);
+ buflen = ZLIB_OUT_SIZE;
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ ahwrite(buf, 1, cnt, AH);
+ }
+
+ free(buf);
+}
-/* Routines that support zlib compressed data I/O */
-#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
-static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
- bool flush);
-static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
-static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
-#endif
-/* Routines that support uncompressed data I/O */
-static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
-static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen);
+static void
+WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ cs->writeF(AH, data, dLen);
+}
+
+static void
+EndCompressorNone(ArchiveHandle *AH, CompressorState *cs)
+{
+ /* no op */
+}
+
+static void
+InitCompressorNone(CompressorState *cs,
+ const pg_compress_specification compression_spec)
+{
+ cs->readData = ReadDataFromArchiveNone;
+ cs->writeData = WriteDataToArchiveNone;
+ cs->end = EndCompressorNone;
+
+ cs->compression_spec = compression_spec;
+}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(const pg_compress_specification compression_spec,
- WriteFunc writeF)
+ ReadFunc readF, WriteFunc writeF)
{
CompressorState *cs;
-#ifndef HAVE_LIBZ
- if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
-
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
+ cs->readF = readF;
cs->writeF = writeF;
- cs->compression_spec = compression_spec;
- /*
- * Perform compression algorithm specific initialization.
- */
-#ifdef HAVE_LIBZ
- if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- InitCompressorZlib(cs, cs->compression_spec.level);
-#endif
-
- return cs;
-}
-
-/*
- * Read all compressed data from the input stream (via readF) and print it
- * out with ahwrite().
- */
-void
-ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF)
-{
switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- ReadDataFromArchiveNone(AH, readF);
+ InitCompressorNone(cs, compression_spec);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
+ InitCompressorGzip(cs, compression_spec);
break;
case PG_COMPRESSION_LZ4:
pg_fatal("compression with %s is not yet supported", "LZ4");
@@ -151,34 +142,8 @@ ReadDataFromArchive(ArchiveHandle *AH,
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
}
-}
-/*
- * Compress and write data to the output stream (via writeF).
- */
-void
-WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen)
-{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- WriteDataToArchiveNone(AH, cs, data, dLen);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ return cs;
}
/*
@@ -187,401 +152,178 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
- switch (cs->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- EndCompressorZlib(AH, cs);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- free(cs);
+ cs->end(AH, cs);
+ pg_free(cs);
}
-/* Private routines, specific to each compression method. */
-
-#ifdef HAVE_LIBZ
-/*
- * Functions for zlib compressed output.
+/*----------------------
+ * Compressed stream API
+ *----------------------
*/
-static void
-InitCompressorZlib(CompressorState *cs, int level)
+static int
+hasSuffix(const char *filename, const char *suffix)
{
- z_streamp zp;
-
- zp = cs->zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- /*
- * zlibOutSize is the buffer size we tell zlib it can output to. We
- * actually allocate one extra byte because some routines want to append a
- * trailing zero byte to the zlib output.
- */
- cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
- cs->zlibOutSize = ZLIB_OUT_SIZE;
-
- if (deflateInit(zp, level) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* Just be paranoid - maybe End is called after Start, with no Write */
- zp->next_out = (void *) cs->zlibOut;
- zp->avail_out = cs->zlibOutSize;
+ int filenamelen = strlen(filename);
+ int suffixlen = strlen(suffix);
+
+ if (filenamelen < suffixlen)
+ return 0;
+
+ return memcmp(&filename[filenamelen - suffixlen],
+ suffix,
+ suffixlen) == 0;
}
+/* free() without changing errno; useful in several places below */
static void
-EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
+free_keep_errno(void *p)
{
- z_streamp zp = cs->zp;
-
- zp->next_in = NULL;
- zp->avail_in = 0;
-
- /* Flush any remaining data from zlib buffer */
- DeflateCompressorZlib(AH, cs, true);
-
- if (deflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression stream: %s", zp->msg);
+ int save_errno = errno;
- free(cs->zlibOut);
- free(cs->zp);
+ free(p);
+ errno = save_errno;
}
-static void
-DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
+/*
+ * Compression None implementation
+ */
+static size_t
+read_none(void *ptr, size_t size, CompressFileHandle *CFH)
{
- z_streamp zp = cs->zp;
- char *out = cs->zlibOut;
- int res = Z_OK;
+ FILE *fp = (FILE *) CFH->private_data;
+ size_t ret;
- while (cs->zp->avail_in != 0 || flush)
- {
- res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
- if (res == Z_STREAM_ERROR)
- pg_fatal("could not compress data: %s", zp->msg);
- if ((flush && (zp->avail_out < cs->zlibOutSize))
- || (zp->avail_out == 0)
- || (zp->avail_in != 0)
- )
- {
- /*
- * Extra paranoia: avoid zero-length chunks, since a zero length
- * chunk is the EOF marker in the custom format. This should never
- * happen but...
- */
- if (zp->avail_out < cs->zlibOutSize)
- {
- /*
- * Any write function should do its own error checking but to
- * make sure we do a check here as well...
- */
- size_t len = cs->zlibOutSize - zp->avail_out;
-
- cs->writeF(AH, out, len);
- }
- zp->next_out = (void *) out;
- zp->avail_out = cs->zlibOutSize;
- }
+ if (size == 0)
+ return 0;
- if (res == Z_STREAM_END)
- break;
- }
+ ret = fread(ptr, 1, size, fp);
+ if (ret != size && !feof(fp))
+ pg_fatal("could not read from input file: %s",
+ strerror(errno));
+
+ return ret;
}
-static void
-WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+static size_t
+write_none(const void *ptr, size_t size, CompressFileHandle *CFH)
{
- cs->zp->next_in = (void *) unconstify(char *, data);
- cs->zp->avail_in = dLen;
- DeflateCompressorZlib(AH, cs, false);
+ return fwrite(ptr, 1, size, (FILE *) CFH->private_data);
}
-static void
-ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
+static const char *
+get_error_none(CompressFileHandle *CFH)
{
- z_streamp zp;
- char *out;
- int res = Z_OK;
- size_t cnt;
- char *buf;
- size_t buflen;
-
- zp = (z_streamp) pg_malloc(sizeof(z_stream));
- zp->zalloc = Z_NULL;
- zp->zfree = Z_NULL;
- zp->opaque = Z_NULL;
-
- buf = pg_malloc(ZLIB_IN_SIZE);
- buflen = ZLIB_IN_SIZE;
-
- out = pg_malloc(ZLIB_OUT_SIZE + 1);
-
- if (inflateInit(zp) != Z_OK)
- pg_fatal("could not initialize compression library: %s",
- zp->msg);
-
- /* no minimal chunk size for zlib */
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- zp->next_in = (void *) buf;
- zp->avail_in = cnt;
-
- while (zp->avail_in > 0)
- {
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
+ return strerror(errno);
+}
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
+static char *
+gets_none(char *ptr, int size, CompressFileHandle *CFH)
+{
+ return fgets(ptr, size, (FILE *) CFH->private_data);
+}
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
- }
- }
+static int
+getc_none(CompressFileHandle *CFH)
+{
+ FILE *fp = (FILE *) CFH->private_data;
+ int ret;
- zp->next_in = NULL;
- zp->avail_in = 0;
- while (res != Z_STREAM_END)
+ ret = fgetc(fp);
+ if (ret == EOF)
{
- zp->next_out = (void *) out;
- zp->avail_out = ZLIB_OUT_SIZE;
- res = inflate(zp, 0);
- if (res != Z_OK && res != Z_STREAM_END)
- pg_fatal("could not uncompress data: %s", zp->msg);
-
- out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
- ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
+ if (!feof(fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
}
- if (inflateEnd(zp) != Z_OK)
- pg_fatal("could not close compression library: %s", zp->msg);
-
- free(buf);
- free(out);
- free(zp);
+ return ret;
}
-#endif /* HAVE_LIBZ */
-
-/*
- * Functions for uncompressed output.
- */
-
-static void
-ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
+static int
+close_none(CompressFileHandle *CFH)
{
- size_t cnt;
- char *buf;
- size_t buflen;
+ FILE *fp = (FILE *) CFH->private_data;
+ int ret = 0;
- buf = pg_malloc(ZLIB_OUT_SIZE);
- buflen = ZLIB_OUT_SIZE;
+ CFH->private_data = NULL;
- while ((cnt = readF(AH, &buf, &buflen)))
- {
- ahwrite(buf, 1, cnt, AH);
- }
+ if (fp)
+ ret = fclose(fp);
- free(buf);
+ return ret;
}
-static void
-WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
- const char *data, size_t dLen)
+
+static int
+eof_none(CompressFileHandle *CFH)
{
- cs->writeF(AH, data, dLen);
+ return feof((FILE *) CFH->private_data);
}
-
-/*----------------------
- * Compressed stream API
- *----------------------
- */
-
-/*
- * cfp represents an open stream, wrapping the underlying FILE or gzFile
- * pointer. This is opaque to the callers.
- */
-struct cfp
+static int
+open_none(const char *path, int fd, const char *mode, CompressFileHandle *CFH)
{
- pg_compress_specification compression_spec;
- void *fp;
-};
+ Assert(CFH->private_data == NULL);
-#ifdef HAVE_LIBZ
-static int hasSuffix(const char *filename, const char *suffix);
-#endif
+ if (fd >= 0)
+ CFH->private_data = fdopen(dup(fd), mode);
+ else
+ CFH->private_data = fopen(path, mode);
-/* free() without changing errno; useful in several places below */
-static void
-free_keep_errno(void *p)
-{
- int save_errno = errno;
+ if (CFH->private_data == NULL)
+ return 1;
- free(p);
- errno = save_errno;
+ return 0;
}
-/*
- * Open a file for reading. 'path' is the file to open, and 'mode' should
- * be either "r" or "rb".
- *
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
- * this will open either "foo" or "foo.gz".
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_read(const char *path, const char *mode)
+static int
+open_write_none(const char *path, const char *mode, CompressFileHandle *CFH)
{
- cfp *fp;
-
- pg_compress_specification compression_spec = {0};
+ Assert(CFH->private_data == NULL);
-#ifdef HAVE_LIBZ
- if (hasSuffix(path, ".gz"))
- {
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(path, mode, compression_spec);
- }
- else
-#endif
- {
- compression_spec.algorithm = PG_COMPRESSION_NONE;
- fp = cfopen(path, mode, compression_spec);
-#ifdef HAVE_LIBZ
- if (fp == NULL)
- {
- char *fname;
+ CFH->private_data = fopen(path, mode);
+ if (CFH->private_data == NULL)
+ return 1;
- fname = psprintf("%s.gz", path);
- compression_spec.algorithm = PG_COMPRESSION_GZIP;
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
- }
-#endif
- }
- return fp;
+ return 0;
}
-/*
- * Open a file for writing. 'path' indicates the path name, and 'mode' must
- * be a filemode as accepted by fopen() and gzopen() that indicates writing
- * ("w", "wb", "a", or "ab").
- *
- * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
- * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
- * 'path' in that case.
- *
- * On failure, return NULL with an error code in errno.
- */
-cfp *
-cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+static void
+InitCompressNone(CompressFileHandle *CFH,
+ const pg_compress_specification compression_spec)
{
- cfp *fp;
-
- if (compression_spec.algorithm == PG_COMPRESSION_NONE)
- fp = cfopen(path, mode, compression_spec);
- else
- {
-#ifdef HAVE_LIBZ
- char *fname;
-
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression_spec);
- free_keep_errno(fname);
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
- fp = NULL; /* keep compiler quiet */
-#endif
- }
- return fp;
+ CFH->open_func = open_none;
+ CFH->open_write_func = open_write_none;
+ CFH->read_func = read_none;
+ CFH->write_func = write_none;
+ CFH->gets_func = gets_none;
+ CFH->getc_func = getc_none;
+ CFH->close_func = close_none;
+ CFH->eof_func = eof_none;
+ CFH->get_error_func = get_error_none;
+
+ CFH->private_data = NULL;
}
/*
- * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
- * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
- * descriptor is not dup'ed and it is the caller's responsibility to do so.
- * The caller must verify that the 'compress_algorithm' is supported by the
- * current build.
- *
- * On failure, return NULL with an error code in errno.
+ * Public interface
*/
-static cfp *
-cfopen_internal(const char *path, int fd, const char *mode,
- pg_compress_specification compression_spec)
+CompressFileHandle *
+InitCompressFileHandle(const pg_compress_specification compression_spec)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ CompressFileHandle *CFH;
- fp->compression_spec = compression_spec;
+ CFH = pg_malloc0(sizeof(CompressFileHandle));
switch (compression_spec.algorithm)
{
case PG_COMPRESSION_NONE:
- if (fd >= 0)
- fp->fp = fdopen(fd, mode);
- else
- fp->fp = fopen(path, mode);
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-
+ InitCompressNone(CFH, compression_spec);
break;
case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- if (compression_spec.level != Z_DEFAULT_COMPRESSION)
- {
- /*
- * user has specified a compression level, so tell zlib to use
- * it
- */
- char mode_compression[32];
-
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression_spec.level);
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode_compression);
- else
- fp->fp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- if (fd >= 0)
- fp->fp = gzdopen(fd, mode);
- else
- fp->fp = gzopen(path, mode);
- }
-
- if (fp->fp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
+ InitCompressGzip(CFH, compression_spec);
break;
case PG_COMPRESSION_LZ4:
pg_fatal("compression with %s is not yet supported", "LZ4");
@@ -591,266 +333,88 @@ cfopen_internal(const char *path, int fd, const char *mode,
break;
}
- return fp;
+ return CFH;
}
-cfp *
-cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec)
+/*
+ * Open a file for reading. 'path' is the file to open, and 'mode' should
+ * be either "r" or "rb".
+ *
+ * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
+ * 'path' doesn't already have it) and try again. So if you pass "foo" as
+ * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
+ * order.
+ *
+ * On failure, return NULL with an error code in errno.
+ */
+CompressFileHandle *
+InitDiscoverCompressFileHandle(const char *path, const char *mode)
{
- return cfopen_internal(path, -1, mode, compression_spec);
-}
+ CompressFileHandle *CFH = NULL;
+ struct stat st;
+ char *fname;
+ pg_compress_specification compression_spec = {0};
-cfp *
-cfdopen(int fd, const char *mode,
- const pg_compress_specification compression_spec)
-{
- return cfopen_internal(NULL, fd, mode, compression_spec);
-}
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
-int
-cfread(void *ptr, int size, cfp *fp)
-{
- int ret = 0;
+ Assert(strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0);
- if (size == 0)
- return 0;
+ fname = strdup(path);
- switch (fp->compression_spec.algorithm)
+ if (hasSuffix(fname, ".gz"))
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ else
{
- case PG_COMPRESSION_NONE:
- ret = fread(ptr, 1, size, (FILE *) fp->fp);
- if (ret != size && !feof((FILE *) fp->fp))
- READ_ERROR_EXIT((FILE *) fp->fp);
+ bool exists;
- break;
- case PG_COMPRESSION_GZIP:
+ exists = (stat(path, &st) == 0);
+ /* avoid unused warning if it is not build with compression */
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_NONE;
#ifdef HAVE_LIBZ
- ret = gzread((gzFile) fp->fp, ptr, size);
- if (ret != size && !gzeof((gzFile) fp->fp))
- {
- int errnum;
- const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
-
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-int
-cfwrite(const void *ptr, int size, cfp *fp)
-{
- int ret = 0;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fwrite(ptr, 1, size, (FILE *) fp->fp);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzwrite((gzFile) fp->fp, ptr, size);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-int
-cfgetc(cfp *fp)
-{
- int ret = 0;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgetc((FILE *) fp->fp);
- if (ret == EOF)
- READ_ERROR_EXIT((FILE *) fp->fp);
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.gz", path);
+ exists = (stat(fname, &st) == 0);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgetc((gzFile) fp->fp);
- if (ret == EOF)
- {
- if (!gzeof((gzFile) fp->fp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ }
#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-char *
-cfgets(cfp *fp, char *buf, int len)
-{
- char *ret = NULL;
-
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = fgets(buf, len, (FILE *) fp->fp);
+#ifdef USE_LZ4
+ if (!exists)
+ {
+ free_keep_errno(fname);
+ fname = psprintf("%s.lz4", path);
+ exists = (stat(fname, &st) == 0);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzgets((gzFile) fp->fp, buf, len);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
+ if (exists)
+ compression_spec.algorithm = PG_COMPRESSION_LZ4;
+ }
#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
-
- return ret;
-}
-
-int
-cfclose(cfp *fp)
-{
- int ret = 0;
-
- if (fp == NULL)
- {
- errno = EBADF;
- return EOF;
}
- switch (fp->compression_spec.algorithm)
+ CFH = InitCompressFileHandle(compression_spec);
+ if (CFH->open_func(fname, -1, mode, CFH))
{
- case PG_COMPRESSION_NONE:
- ret = fclose((FILE *) fp->fp);
- fp->fp = NULL;
-
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzclose((gzFile) fp->fp);
- fp->fp = NULL;
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
+ free_keep_errno(CFH);
+ CFH = NULL;
}
+ free_keep_errno(fname);
- free_keep_errno(fp);
-
- return ret;
+ return CFH;
}
int
-cfeof(cfp *fp)
+DestroyCompressFileHandle(CompressFileHandle *CFH)
{
int ret = 0;
- switch (fp->compression_spec.algorithm)
- {
- case PG_COMPRESSION_NONE:
- ret = feof((FILE *) fp->fp);
+ if (CFH->private_data)
+ ret = CFH->close_func(CFH);
- break;
- case PG_COMPRESSION_GZIP:
-#ifdef HAVE_LIBZ
- ret = gzeof((gzFile) fp->fp);
-#else
- pg_fatal("this build does not support compression with %s",
- "gzip");
-#endif
- break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
- case PG_COMPRESSION_ZSTD:
- pg_fatal("compression with %s is not yet supported", "ZSTD");
- break;
- }
+ free_keep_errno(CFH);
return ret;
}
-
-const char *
-get_cfp_error(cfp *fp)
-{
- if (fp->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- {
-#ifdef HAVE_LIBZ
- int errnum;
- const char *errmsg = gzerror((gzFile) fp->fp, &errnum);
-
- if (errnum != Z_ERRNO)
- return errmsg;
-#else
- pg_fatal("this build does not support compression with %s", "gzip");
-#endif
- }
-
- return strerror(errno);
-}
-
-#ifdef HAVE_LIBZ
-static int
-hasSuffix(const char *filename, const char *suffix)
-{
- int filenamelen = strlen(filename);
- int suffixlen = strlen(suffix);
-
- if (filenamelen < suffixlen)
- return 0;
-
- return memcmp(&filename[filenamelen - suffixlen],
- suffix,
- suffixlen) == 0;
-}
-
-#endif
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index 6fad6c2cd5..62e3da1b1d 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -37,34 +37,63 @@ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
*/
typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
-/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
+struct CompressorState
+{
+ /*
+ * Read all compressed data from the input stream (via readF) and print it
+ * out with ahwrite().
+ */
+ void (*readData) (ArchiveHandle *AH, CompressorState *cs);
+
+ /*
+ * Compress and write data to the output stream (via writeF).
+ */
+ void (*writeData) (ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+ void (*end) (ArchiveHandle *AH, CompressorState *cs);
+
+ ReadFunc readF;
+ WriteFunc writeF;
+
+ pg_compress_specification compression_spec;
+ void *private_data;
+};
extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+ ReadFunc readF,
WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH,
- const pg_compress_specification compression_spec,
- ReadFunc readF);
-extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
- const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
+/*
+ * Compress File Handle
+ */
+typedef struct CompressFileHandle CompressFileHandle;
+
+struct CompressFileHandle
+{
+ int (*open_func) (const char *path, int fd, const char *mode,
+ CompressFileHandle *CFH);
+ int (*open_write_func) (const char *path, const char *mode,
+ CompressFileHandle *cxt);
+ size_t (*read_func) (void *ptr, size_t size, CompressFileHandle *CFH);
+ size_t (*write_func) (const void *ptr, size_t size,
+ struct CompressFileHandle *CFH);
+ char *(*gets_func) (char *s, int size, CompressFileHandle *CFH);
+ int (*getc_func) (CompressFileHandle *CFH);
+ int (*eof_func) (CompressFileHandle *CFH);
+ int (*close_func) (CompressFileHandle *CFH);
+ const char *(*get_error_func) (CompressFileHandle *CFH);
+
+ pg_compress_specification compression_spec;
+ void *private_data;
+};
-typedef struct cfp cfp;
+extern CompressFileHandle *InitCompressFileHandle(
+ const pg_compress_specification compression_spec);
-extern cfp *cfopen(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern cfp *cfdopen(int fd, const char *mode,
- pg_compress_specification compression_spec);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode,
- const pg_compress_specification compression_spec);
-extern int cfread(void *ptr, int size, cfp *fp);
-extern int cfwrite(const void *ptr, int size, cfp *fp);
-extern int cfgetc(cfp *fp);
-extern char *cfgets(cfp *fp, char *buf, int len);
-extern int cfclose(cfp *fp);
-extern int cfeof(cfp *fp);
-extern const char *get_cfp_error(cfp *fp);
+extern CompressFileHandle *InitDiscoverCompressFileHandle(const char *path,
+ const char *mode);
+extern int DestroyCompressFileHandle(CompressFileHandle *CFH);
#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 5537cda3cc..714e492c60 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -2,6 +2,7 @@
pg_dump_common_sources = files(
'compress_io.c',
+ 'compress_gzip.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index fb94317ad9..1f207c6f4d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -95,8 +95,8 @@ static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
static void SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec);
-static cfp *SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
+static CompressFileHandle *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -272,7 +272,7 @@ CloseArchive(Archive *AHX)
/* Close the output */
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -355,7 +355,7 @@ RestoreArchive(Archive *AHX)
bool parallel_mode;
bool supports_compression;
TocEntry *te;
- cfp *sav;
+ CompressFileHandle *sav;
AH->stage = STAGE_INITIALIZING;
@@ -1127,7 +1127,7 @@ PrintTOCSummary(Archive *AHX)
TocEntry *te;
pg_compress_specification out_compression_spec = {0};
teSection curSection;
- cfp *sav;
+ CompressFileHandle *sav;
const char *fmtName;
char stamp_str[64];
@@ -1143,9 +1143,10 @@ PrintTOCSummary(Archive *AHX)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression_spec.level);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compression_spec.algorithm));
switch (AH->format)
{
@@ -1502,6 +1503,7 @@ static void
SetOutput(ArchiveHandle *AH, const char *filename,
const pg_compress_specification compression_spec)
{
+ CompressFileHandle *CFH;
const char *mode;
int fn = -1;
@@ -1524,33 +1526,32 @@ SetOutput(ArchiveHandle *AH, const char *filename,
else
mode = PG_BINARY_W;
- if (fn >= 0)
- AH->OF = cfdopen(dup(fn), mode, compression_spec);
- else
- AH->OF = cfopen(filename, mode, compression_spec);
+ CFH = InitCompressFileHandle(compression_spec);
- if (!AH->OF)
+ if (CFH->open_func(filename, fn, mode, CFH))
{
if (filename)
pg_fatal("could not open output file \"%s\": %m", filename);
else
pg_fatal("could not open output file: %m");
}
+
+ AH->OF = CFH;
}
-static cfp *
+static CompressFileHandle *
SaveOutput(ArchiveHandle *AH)
{
- return (cfp *) AH->OF;
+ return (CompressFileHandle *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
+RestoreOutput(ArchiveHandle *AH, CompressFileHandle *savedOutput)
{
int res;
errno = 0;
- res = cfclose(AH->OF);
+ res = DestroyCompressFileHandle(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -1689,7 +1690,11 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
else if (RestoringToDB(AH))
bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
+ {
+ CompressFileHandle *CFH = (CompressFileHandle *) AH->OF;
+
+ bytes_written = CFH->write_func(ptr, size * nmemb, CFH);
+ }
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2031,6 +2036,18 @@ ReadStr(ArchiveHandle *AH)
return buf;
}
+static bool
+_fileExistsInDirectory(const char *dir, const char *filename)
+{
+ struct stat st;
+ char buf[MAXPGPATH];
+
+ if (snprintf(buf, MAXPGPATH, "%s/%s", dir, filename) >= MAXPGPATH)
+ pg_fatal("directory name too long: \"%s\"", dir);
+
+ return (stat(buf, &st) == 0 && S_ISREG(st.st_mode));
+}
+
static int
_discoverArchiveFormat(ArchiveHandle *AH)
{
@@ -2061,26 +2078,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
- char buf[MAXPGPATH];
-
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat"))
return AH->format;
- }
-
#ifdef HAVE_LIBZ
- if (snprintf(buf, MAXPGPATH, "%s/toc.dat.gz", AH->fSpec) >= MAXPGPATH)
- pg_fatal("directory name too long: \"%s\"",
- AH->fSpec);
- if (stat(buf, &st) == 0 && S_ISREG(st.st_mode))
- {
- AH->format = archDirectory;
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
- }
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -2178,6 +2181,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ CompressFileHandle *CFH;
pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
@@ -2233,7 +2237,10 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
/* Open stdout with no compression for AH output handle */
out_compress_spec.algorithm = PG_COMPRESSION_NONE;
- AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
+ CFH = InitCompressFileHandle(out_compress_spec);
+ if (CFH->open_func(NULL, fileno(stdout), PG_BINARY_A, CFH))
+ pg_fatal("could not open stdout for appending: %m");
+ AH->OF = CFH;
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -3646,12 +3653,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- /*
- * For now the compression type is implied by the level. This will need
- * to change once support for more compression algorithms is added,
- * requiring a format bump.
- */
- WriteInt(AH, AH->compression_spec.level);
+ AH->WriteBytePtr(AH, AH->compression_spec.algorithm);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3722,10 +3724,11 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
- /* Guess the compression method based on the level */
- AH->compression_spec.algorithm = PG_COMPRESSION_NONE;
- if (AH->version >= K_VERS_1_2)
+ if (AH->version >= K_VERS_1_15)
+ AH->compression_spec.algorithm = AH->ReadBytePtr(AH);
+ else if (AH->version >= K_VERS_1_2)
{
+ /* Guess the compression method based on the level */
if (AH->version < K_VERS_1_4)
AH->compression_spec.level = AH->ReadBytePtr(AH);
else
@@ -3737,10 +3740,17 @@ ReadHead(ArchiveHandle *AH)
else
AH->compression_spec.algorithm = PG_COMPRESSION_GZIP;
+ if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
+ {
+ bool unsupported = false;
+
#ifndef HAVE_LIBZ
- if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
- pg_fatal("archive is compressed, but this installation does not support compression");
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+ unsupported = true;
#endif
+ if (unsupported)
+ pg_fatal("archive is compressed, but this installation does not support compression");
+ }
if (AH->version >= K_VERS_1_4)
{
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 4725e49747..18b38c17ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -65,10 +65,13 @@
#define K_VERS_1_13 MAKE_ARCHIVE_VERSION(1, 13, 0) /* change search_path
* behavior */
#define K_VERS_1_14 MAKE_ARCHIVE_VERSION(1, 14, 0) /* add tableam */
+#define K_VERS_1_15 MAKE_ARCHIVE_VERSION(1, 15, 0) /* add
+ * compression_algorithm
+ * in header */
/* Current archive version number (the format we can output) */
#define K_VERS_MAJOR 1
-#define K_VERS_MINOR 14
+#define K_VERS_MINOR 15
#define K_VERS_REV 0
#define K_VERS_SELF MAKE_ARCHIVE_VERSION(K_VERS_MAJOR, K_VERS_MINOR, K_VERS_REV)
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index d1e54644a9..512ab043af 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -317,15 +319,15 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
CompressorState *cs = ctx->cs;
if (dLen > 0)
- /* WriteDataToArchive() internally throws write errors */
- WriteDataToArchive(AH, cs, data, dLen);
+ /* writeData() internally throws write errors */
+ cs->writeData(AH, cs, data, dLen);
}
/*
* Called by the archiver when a dumper's 'DataDumper' routine has
* finished.
*
- * Optional.
+ * Mandatory.
*/
static void
_EndData(ArchiveHandle *AH, TocEntry *te)
@@ -333,6 +335,8 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
+ ctx->cs = NULL;
+
/* Send the end marker */
WriteInt(AH, 0);
}
@@ -377,7 +381,9 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression_spec, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compression_spec,
+ NULL,
+ _CustomWriteFunc);
}
/*
@@ -566,7 +572,12 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression_spec, _CustomReadFunc);
+ CompressorState *cs;
+
+ cs = AllocateCompressor(AH->compression_spec,
+ _CustomReadFunc, NULL);
+ cs->readData(AH, cs);
+ EndCompressor(AH, cs);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index f6aee775eb..b6d025576f 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -50,9 +50,8 @@ typedef struct
*/
char *directory;
- cfp *dataFH; /* currently open data file */
-
- cfp *LOsTocFH; /* file handle for blobs.toc */
+ CompressFileHandle *dataFH; /* currently open data file */
+ CompressFileHandle *LOsTocFH; /* file handle for blobs.toc */
ParallelState *pstate; /* for parallel backup / restore */
} lclContext;
@@ -198,11 +197,11 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
else
{ /* Read Mode */
char fname[MAXPGPATH];
- cfp *tocFH;
+ CompressFileHandle *tocFH;
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
if (tocFH == NULL)
pg_fatal("could not open input file \"%s\": %m", fname);
@@ -218,7 +217,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
ReadToc(AH);
/* Nothing else in the file, so close it again... */
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
ctx->dataFH = NULL;
}
@@ -327,9 +326,9 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
- AH->compression_spec);
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+
+ if (ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -346,15 +345,16 @@ static void
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
+ if (dLen > 0 && CFH->write_func(data, dLen, CFH) != dLen)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error_func(CFH));
}
}
@@ -370,7 +370,7 @@ _EndData(ArchiveHandle *AH, TocEntry *te)
lclContext *ctx = (lclContext *) AH->formatData;
/* Close the file */
- if (cfclose(ctx->dataFH) != 0)
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
pg_fatal("could not close data file: %m");
ctx->dataFH = NULL;
@@ -385,26 +385,25 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
size_t cnt;
char *buf;
size_t buflen;
- cfp *cfp;
+ CompressFileHandle *CFH;
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
-
- if (!cfp)
+ CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
+ if (!CFH)
pg_fatal("could not open input file \"%s\": %m", filename);
buf = pg_malloc(ZLIB_OUT_SIZE);
buflen = ZLIB_OUT_SIZE;
- while ((cnt = cfread(buf, buflen, cfp)))
+ while ((cnt = CFH->read_func(buf, buflen, CFH)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
- if (cfclose(cfp) != 0)
+ if (DestroyCompressFileHandle(CFH) != 0)
pg_fatal("could not close data file \"%s\": %m", filename);
}
@@ -435,6 +434,7 @@ _LoadLOs(ArchiveHandle *AH)
{
Oid oid;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH;
char tocfname[MAXPGPATH];
char line[MAXPGPATH];
@@ -442,14 +442,14 @@ _LoadLOs(ArchiveHandle *AH)
setFilePath(AH, tocfname, "blobs.toc");
- ctx->LOsTocFH = cfopen_read(tocfname, PG_BINARY_R);
+ CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
if (ctx->LOsTocFH == NULL)
pg_fatal("could not open large object TOC file \"%s\" for input: %m",
tocfname);
/* Read the LOs TOC file line-by-line, and process each LO */
- while ((cfgets(ctx->LOsTocFH, line, MAXPGPATH)) != NULL)
+ while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
{
char lofname[MAXPGPATH + 1];
char path[MAXPGPATH];
@@ -464,11 +464,11 @@ _LoadLOs(ArchiveHandle *AH)
_PrintFileData(AH, path);
EndRestoreLO(AH, oid);
}
- if (!cfeof(ctx->LOsTocFH))
+ if (!CFH->eof_func(CFH))
pg_fatal("error reading large object TOC file \"%s\"",
tocfname);
- if (cfclose(ctx->LOsTocFH) != 0)
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
pg_fatal("could not close large object TOC file \"%s\": %m",
tocfname);
@@ -488,15 +488,16 @@ _WriteByte(ArchiveHandle *AH, const int i)
{
unsigned char c = (unsigned char) i;
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(&c, 1, ctx->dataFH) != 1)
+ if (CFH->write_func(&c, 1, CFH) != 1)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error_func(CFH));
}
return 1;
@@ -512,8 +513,9 @@ static int
_ReadByte(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
- return cfgetc(ctx->dataFH);
+ return CFH->getc_func(CFH);
}
/*
@@ -524,15 +526,16 @@ static void
_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
errno = 0;
- if (cfwrite(buf, len, ctx->dataFH) != len)
+ if (CFH->write_func(buf, len, CFH) != len)
{
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0)
errno = ENOSPC;
pg_fatal("could not write to output file: %s",
- get_cfp_error(ctx->dataFH));
+ CFH->get_error_func(CFH));
}
}
@@ -545,12 +548,13 @@ static void
_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->dataFH;
/*
- * If there was an I/O error, we already exited in cfread(), so here we
+ * If there was an I/O error, we already exited in readF(), so here we
* exit on short reads.
*/
- if (cfread(buf, len, ctx->dataFH) != len)
+ if (CFH->read_func(buf, len, CFH) != len)
pg_fatal("could not read from input file: end of file");
}
@@ -573,7 +577,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
- cfp *tocFH;
+ CompressFileHandle *tocFH;
pg_compress_specification compression_spec = {0};
char fname[MAXPGPATH];
@@ -584,8 +588,8 @@ _CloseArchive(ArchiveHandle *AH)
/* The TOC is always created uncompressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- tocFH = cfopen_write(fname, PG_BINARY_W, compression_spec);
- if (tocFH == NULL)
+ tocFH = InitCompressFileHandle(compression_spec);
+ if (tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -598,7 +602,7 @@ _CloseArchive(ArchiveHandle *AH)
WriteHead(AH);
AH->format = archDirectory;
WriteToc(AH);
- if (cfclose(tocFH) != 0)
+ if (DestroyCompressFileHandle(tocFH) != 0)
pg_fatal("could not close TOC file: %m");
WriteDataChunks(AH, ctx->pstate);
@@ -649,8 +653,8 @@ _StartLOs(ArchiveHandle *AH, TocEntry *te)
/* The LO TOC file is never compressed */
compression_spec.algorithm = PG_COMPRESSION_NONE;
- ctx->LOsTocFH = cfopen_write(fname, "ab", compression_spec);
- if (ctx->LOsTocFH == NULL)
+ ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
+ if (ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -667,9 +671,8 @@ _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression_spec);
-
- if (ctx->dataFH == NULL)
+ ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
+ if (ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -682,18 +685,19 @@ static void
_EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ CompressFileHandle *CFH = ctx->LOsTocFH;
char buf[50];
int len;
- /* Close the LO data file itself */
- if (cfclose(ctx->dataFH) != 0)
- pg_fatal("could not close LO data file: %m");
+ /* Close the BLOB data file itself */
+ if (DestroyCompressFileHandle(ctx->dataFH) != 0)
+ pg_fatal("could not close blob data file: %m");
ctx->dataFH = NULL;
/* register the LO in blobs.toc */
len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
- if (cfwrite(buf, len, ctx->LOsTocFH) != len)
- pg_fatal("could not write to LOs TOC file");
+ if (CFH->write_func(buf, len, CFH) != len)
+ pg_fatal("could not write to blobs TOC file");
}
/*
@@ -706,8 +710,8 @@ _EndLOs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
- if (cfclose(ctx->LOsTocFH) != 0)
- pg_fatal("could not close LOs TOC file: %m");
+ if (DestroyCompressFileHandle(ctx->LOsTocFH) != 0)
+ pg_fatal("could not close blobs TOC file: %m");
ctx->LOsTocFH = NULL;
}
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 7c3067a3f4..1110ffa5e2 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -94,7 +94,7 @@ my %pgdump_runs = (
command => [
'pg_restore', '-l', "$tempdir/compression_gzip_custom.dump",
],
- expected => qr/Compression: 1/,
+ expected => qr/Compression: gzip/,
name => 'data content is gzip-compressed'
},
},
@@ -239,8 +239,8 @@ my %pgdump_runs = (
command =>
[ 'pg_restore', '-l', "$tempdir/defaults_custom_format.dump", ],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip-compressed by default if available',
},
},
@@ -264,8 +264,8 @@ my %pgdump_runs = (
command =>
[ 'pg_restore', '-l', "$tempdir/defaults_dir_format", ],
expected => $supports_gzip ?
- qr/Compression: -1/ :
- qr/Compression: 0/,
+ qr/Compression: gzip/ :
+ qr/Compression: none/,
name => 'data content is gzip-compressed by default',
},
glob_patterns => [
diff --git a/src/tools/pginclude/cpluspluscheck b/src/tools/pginclude/cpluspluscheck
index b393f2a2ea..8805237edb 100755
--- a/src/tools/pginclude/cpluspluscheck
+++ b/src/tools/pginclude/cpluspluscheck
@@ -150,6 +150,7 @@ do
# pg_dump is not C++-clean because it uses "public" and "namespace"
# as field names, which is unfortunate but we won't change it now.
+ test "$f" = src/bin/pg_dump/compress_gzip.h && continue
test "$f" = src/bin/pg_dump/compress_io.h && continue
test "$f" = src/bin/pg_dump/parallel.h && continue
test "$f" = src/bin/pg_dump/pg_backup_archiver.h && continue
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 60c71d05fe..81a451641a 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -428,6 +428,7 @@ CompiledExprState
CompositeIOData
CompositeTypeStmt
CompoundAffixFlag
+CompressFileHandle
CompressionLocation
CompressorState
ComputeXidHorizonsResult
@@ -1034,6 +1035,7 @@ GucStack
GucStackState
GucStringAssignHook
GucStringCheckHook
+GzipCompressorState
HANDLE
HASHACTION
HASHBUCKET
--
2.34.1
[text/x-patch] v19-0003-Add-LZ4-compression-in-pg_-dump-restore.patch (28.8K, ../../f8Wmj1EmRT7CHBfMcg0jM_Wts_hyV22WsLWzf5X5KImQo_5CtZXHK3UjxL8Wne6g7zSccwXLNToLCINfEIPbNni53gKO17AuHplMjm-XBmA=@pm.me/4-v19-0003-Add-LZ4-compression-in-pg_-dump-restore.patch)
download | inline diff:
From e211681ca4d5d770d503cee9f5a4c7d44f2b1444 Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <[email protected]>
Date: Wed, 21 Dec 2022 09:49:36 +0000
Subject: [PATCH v19 3/3] Add LZ4 compression in pg_{dump|restore}
Within compress_lz4.{c,h} the streaming API and a file API compression is
implemented.. The first one, is aimed at inlined use cases and thus simple
lz4.h calls can be used directly. The second one is generating output, or is
parsing input, which can be read/generated via the lz4 utility.
Wherever the LZ4F api does not implement all the functionality corresponding
to fread(), fwrite(), fgets(), fgetc(), feof(), and fclose(), it has been
implemented localy.
---
doc/src/sgml/ref/pg_dump.sgml | 13 +-
src/bin/pg_dump/Makefile | 2 +
src/bin/pg_dump/compress_io.c | 11 +-
src/bin/pg_dump/compress_lz4.c | 618 +++++++++++++++++++++++++++
src/bin/pg_dump/compress_lz4.h | 22 +
src/bin/pg_dump/meson.build | 8 +-
src/bin/pg_dump/pg_backup_archiver.c | 14 +-
src/bin/pg_dump/pg_dump.c | 5 +-
src/bin/pg_dump/t/002_pg_dump.pl | 82 +++-
src/tools/pginclude/cpluspluscheck | 1 +
src/tools/pgindent/typedefs.list | 1 +
11 files changed, 756 insertions(+), 21 deletions(-)
create mode 100644 src/bin/pg_dump/compress_lz4.c
create mode 100644 src/bin/pg_dump/compress_lz4.h
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 2c938cd7e1..49d218905f 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -330,9 +330,10 @@ PostgreSQL documentation
machine-readable format that <application>pg_restore</application>
can read. A directory format archive can be manipulated with
standard Unix tools; for example, files in an uncompressed archive
- can be compressed with the <application>gzip</application> tool.
- This format is compressed by default and also supports parallel
- dumps.
+ can be compressed with the <application>gzip</application> or
+ <application>lz4</application>tool.
+ This format is compressed by default using <literal>gzip</literal>
+ and also supports parallel dumps.
</para>
</listitem>
</varlistentry>
@@ -654,7 +655,7 @@ PostgreSQL documentation
<para>
Specify the compression method and/or the compression level to use.
The compression method can be set to <literal>gzip</literal> or
- <literal>none</literal> for no compression.
+ <literal>lz4</literal> or <literal>none</literal> for no compression.
A compression detail string can optionally be specified. If the
detail string is an integer, it specifies the compression level.
Otherwise, it should be a comma-separated list of items, each of the
@@ -675,8 +676,8 @@ PostgreSQL documentation
individual table-data segments, and the default is to compress using
<literal>gzip</literal> at a moderate level. For plain text output,
setting a nonzero compression level causes the entire output file to be compressed,
- as though it had been fed through <application>gzip</application>; but the default
- is not to compress.
+ as though it had been fed through <application>gzip</application> or
+ <application>lz4</application>; but the default is not to compress.
</para>
<para>
The tar archive format currently does not support compression at all.
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 29eab02d37..28c1fc27cc 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -17,6 +17,7 @@ top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
export GZIP_PROGRAM=$(GZIP)
+export LZ4
export with_icu
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
@@ -25,6 +26,7 @@ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
OBJS = \
$(WIN32RES) \
compress_gzip.o \
+ compress_lz4.o \
compress_io.o \
dumputils.o \
parallel.o \
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6975ea6920..81760600fc 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -60,6 +60,7 @@
#include "compress_io.h"
#include "compress_gzip.h"
+#include "compress_lz4.h"
#include "pg_backup_utils.h"
#ifdef HAVE_LIBZ
@@ -136,7 +137,7 @@ AllocateCompressor(const pg_compress_specification compression_spec,
InitCompressorGzip(cs, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
+ InitCompressorLZ4(cs, compression_spec);
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
@@ -326,7 +327,7 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
InitCompressGzip(CFH, compression_spec);
break;
case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
+ InitCompressLZ4(CFH, compression_spec);
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
@@ -340,12 +341,12 @@ InitCompressFileHandle(const pg_compress_specification compression_spec)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the "{.gz,.lz4}" suffix (i
+ * If the file at 'path' does not exist, we append the ".gz" suffix (if
* 'path' doesn't already have it) and try again. So if you pass "foo" as
- * 'path', this will open either "foo" or "foo.gz" or "foo.lz4", trying in that
- * order.
+ * 'path', this will open either "foo" or "foo.gz", trying in that order.
*
* On failure, return NULL with an error code in errno.
+ *
*/
CompressFileHandle *
InitDiscoverCompressFileHandle(const char *path, const char *mode)
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
new file mode 100644
index 0000000000..c97e16187a
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -0,0 +1,618 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_lz4.c
+ * Routines for archivers to write an uncompressed or compressed data
+ * stream.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_lz4.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+#include "pg_backup_utils.h"
+
+#include "compress_lz4.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#include <lz4frame.h>
+
+#define LZ4_OUT_SIZE (4 * 1024)
+#define LZ4_IN_SIZE (16 * 1024)
+
+/*
+ * LZ4F_HEADER_SIZE_MAX first appeared in v1.7.5 of the library.
+ * Redefine it for installations with a lesser version.
+ */
+#ifndef LZ4F_HEADER_SIZE_MAX
+#define LZ4F_HEADER_SIZE_MAX 32
+#endif
+
+/*----------------------
+ * Compressor API
+ *----------------------
+ */
+
+typedef struct LZ4CompressorState
+{
+ char *outbuf;
+ size_t outsize;
+} LZ4CompressorState;
+
+/* Private routines that support LZ4 compressed data I/O */
+static void ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs);
+static void WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen);
+static void EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs);
+
+static void
+ReadDataFromArchiveLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4_streamDecode_t lz4StreamDecode;
+ char *buf;
+ char *decbuf;
+ size_t buflen;
+ size_t cnt;
+
+ buflen = LZ4_IN_SIZE;
+ buf = pg_malloc(buflen);
+ decbuf = pg_malloc(buflen);
+
+ LZ4_setStreamDecode(&lz4StreamDecode, NULL, 0);
+
+ while ((cnt = cs->readF(AH, &buf, &buflen)))
+ {
+ int decBytes = LZ4_decompress_safe_continue(&lz4StreamDecode,
+ buf, decbuf,
+ cnt, buflen);
+
+ ahwrite(decbuf, 1, decBytes, AH);
+ }
+
+ pg_free(buf);
+ pg_free(decbuf);
+}
+
+static void
+WriteDataToArchiveLZ4(ArchiveHandle *AH, CompressorState *cs,
+ const void *data, size_t dLen)
+{
+ LZ4CompressorState *LZ4cs = (LZ4CompressorState *) cs->private_data;
+ size_t compressed;
+ size_t requiredsize = LZ4_compressBound(dLen);
+
+ if (requiredsize > LZ4cs->outsize)
+ {
+ LZ4cs->outbuf = pg_realloc(LZ4cs->outbuf, requiredsize);
+ LZ4cs->outsize = requiredsize;
+ }
+
+ compressed = LZ4_compress_default(data, LZ4cs->outbuf,
+ dLen, LZ4cs->outsize);
+
+ if (compressed <= 0)
+ pg_fatal("failed to LZ4 compress data");
+
+ cs->writeF(AH, LZ4cs->outbuf, compressed);
+}
+
+static void
+EndCompressorLZ4(ArchiveHandle *AH, CompressorState *cs)
+{
+ LZ4CompressorState *LZ4cs;
+
+ LZ4cs = (LZ4CompressorState *) cs->private_data;
+ if (LZ4cs)
+ {
+ pg_free(LZ4cs->outbuf);
+ pg_free(LZ4cs);
+ cs->private_data = NULL;
+ }
+}
+
+
+/* Public routines that support LZ4 compressed data I/O */
+void
+InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ cs->readData = ReadDataFromArchiveLZ4;
+ cs->writeData = WriteDataToArchiveLZ4;
+ cs->end = EndCompressorLZ4;
+
+ cs->compression_spec = compression_spec;
+
+ /* Will be lazy init'd */
+ cs->private_data = pg_malloc0(sizeof(LZ4CompressorState));
+}
+
+/*----------------------
+ * Compress File API
+ *----------------------
+ */
+
+/*
+ * State needed for LZ4 (de)compression using the CompressFileHandle API.
+ */
+typedef struct LZ4File
+{
+ FILE *fp;
+
+ LZ4F_preferences_t prefs;
+
+ LZ4F_compressionContext_t ctx;
+ LZ4F_decompressionContext_t dtx;
+
+ bool inited;
+ bool compressing;
+
+ size_t buflen;
+ char *buffer;
+
+ size_t overflowalloclen;
+ size_t overflowlen;
+ char *overflowbuf;
+
+ size_t errcode;
+} LZ4File;
+
+/*
+ * LZ4 equivalent to feof() or gzeof(). The end of file
+ * is reached if there is no decompressed output in the
+ * overflow buffer and the end of the file is reached.
+ */
+static int
+LZ4File_eof(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+
+ return fs->overflowlen == 0 && feof(fs->fp);
+}
+
+static const char *
+LZ4File_get_error(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ const char *errmsg;
+
+ if (LZ4F_isError(fs->errcode))
+ errmsg = LZ4F_getErrorName(fs->errcode);
+ else
+ errmsg = strerror(errno);
+
+ return errmsg;
+}
+
+/*
+ * Prepare an already alloc'ed LZ4File struct for subsequent calls.
+ *
+ * It creates the nessary contexts for the operations. When compressing,
+ * it additionally writes the LZ4 header in the output stream.
+ */
+static int
+LZ4File_init(LZ4File * fs, int size, bool compressing)
+{
+ size_t status;
+
+ if (fs->inited)
+ return 0;
+
+ fs->compressing = compressing;
+ fs->inited = true;
+
+ if (fs->compressing)
+ {
+ fs->buflen = LZ4F_compressBound(LZ4_IN_SIZE, &fs->prefs);
+ if (fs->buflen < LZ4F_HEADER_SIZE_MAX)
+ fs->buflen = LZ4F_HEADER_SIZE_MAX;
+
+ status = LZ4F_createCompressionContext(&fs->ctx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buffer = pg_malloc(fs->buflen);
+ status = LZ4F_compressBegin(fs->ctx, fs->buffer, fs->buflen,
+ &fs->prefs);
+
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+ else
+ {
+ status = LZ4F_createDecompressionContext(&fs->dtx, LZ4F_VERSION);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return 1;
+ }
+
+ fs->buflen = size > LZ4_OUT_SIZE ? size : LZ4_OUT_SIZE;
+ fs->buffer = pg_malloc(fs->buflen);
+
+ fs->overflowalloclen = fs->buflen;
+ fs->overflowbuf = pg_malloc(fs->overflowalloclen);
+ fs->overflowlen = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Read already decompressed content from the overflow buffer into 'ptr' up to
+ * 'size' bytes, if available. If the eol_flag is set, then stop at the first
+ * occurance of the new line char prior to 'size' bytes.
+ *
+ * Any unread content in the overflow buffer, is moved to the beginning.
+ */
+static int
+LZ4File_read_overflow(LZ4File * fs, void *ptr, int size, bool eol_flag)
+{
+ char *p;
+ int readlen = 0;
+
+ if (fs->overflowlen == 0)
+ return 0;
+
+ if (fs->overflowlen >= size)
+ readlen = size;
+ else
+ readlen = fs->overflowlen;
+
+ if (eol_flag && (p = memchr(fs->overflowbuf, '\n', readlen)))
+ /* Include the line terminating char */
+ readlen = p - fs->overflowbuf + 1;
+
+ memcpy(ptr, fs->overflowbuf, readlen);
+ fs->overflowlen -= readlen;
+
+ if (fs->overflowlen > 0)
+ memmove(fs->overflowbuf, fs->overflowbuf + readlen, fs->overflowlen);
+
+ return readlen;
+}
+
+/*
+ * The workhorse for reading decompressed content out of an LZ4 compressed
+ * stream.
+ *
+ * It will read up to 'ptrsize' decompressed content, or up to the new line char
+ * if found first when the eol_flag is set. It is possible that the decompressed
+ * output generated by reading any compressed input via the LZ4F API, exceeds
+ * 'ptrsize'. Any exceeding decompressed content is stored at an overflow
+ * buffer within LZ4File. Of course, when the function is called, it will first
+ * try to consume any decompressed content already present in the overflow
+ * buffer, before decompressing new content.
+ */
+static int
+LZ4File_read_internal(LZ4File * fs, void *ptr, int ptrsize, bool eol_flag)
+{
+ size_t dsize = 0;
+ size_t rsize;
+ size_t size = ptrsize;
+ bool eol_found = false;
+
+ void *readbuf;
+
+ /* Lazy init */
+ if (!fs->inited && LZ4File_init(fs, size, false /* decompressing */ ))
+ return -1;
+
+ /* Verfiy that there is enough space in the outbuf */
+ if (size > fs->buflen)
+ {
+ fs->buflen = size;
+ fs->buffer = pg_realloc(fs->buffer, size);
+ }
+
+ /* use already decompressed content if available */
+ dsize = LZ4File_read_overflow(fs, ptr, size, eol_flag);
+ if (dsize == size || (eol_flag && memchr(ptr, '\n', dsize)))
+ return dsize;
+
+ readbuf = pg_malloc(size);
+
+ do
+ {
+ char *rp;
+ char *rend;
+
+ rsize = fread(readbuf, 1, size, fs->fp);
+ if (rsize < size && !feof(fs->fp))
+ return -1;
+
+ rp = (char *) readbuf;
+ rend = (char *) readbuf + rsize;
+
+ while (rp < rend)
+ {
+ size_t status;
+ size_t outlen = fs->buflen;
+ size_t read_remain = rend - rp;
+
+ memset(fs->buffer, 0, outlen);
+ status = LZ4F_decompress(fs->dtx, fs->buffer, &outlen,
+ rp, &read_remain, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ rp += read_remain;
+
+ /*
+ * fill in what space is available in ptr if the eol flag is set,
+ * either skip if one already found or fill up to EOL if present
+ * in the outbuf
+ */
+ if (outlen > 0 && dsize < size && eol_found == false)
+ {
+ char *p;
+ size_t lib = (eol_flag == 0) ? size - dsize : size - 1 - dsize;
+ size_t len = outlen < lib ? outlen : lib;
+
+ if (eol_flag == true &&
+ (p = memchr(fs->buffer, '\n', outlen)) &&
+ (size_t) (p - fs->buffer + 1) <= len)
+ {
+ len = p - fs->buffer + 1;
+ eol_found = true;
+ }
+
+ memcpy((char *) ptr + dsize, fs->buffer, len);
+ dsize += len;
+
+ /* move what did not fit, if any, at the begining of the buf */
+ if (len < outlen)
+ memmove(fs->buffer, fs->buffer + len, outlen - len);
+ outlen -= len;
+ }
+
+ /* if there is available output, save it */
+ if (outlen > 0)
+ {
+ while (fs->overflowlen + outlen > fs->overflowalloclen)
+ {
+ fs->overflowalloclen *= 2;
+ fs->overflowbuf = pg_realloc(fs->overflowbuf,
+ fs->overflowalloclen);
+ }
+
+ memcpy(fs->overflowbuf + fs->overflowlen, fs->buffer, outlen);
+ fs->overflowlen += outlen;
+ }
+ }
+ } while (rsize == size && dsize < size && eol_found == 0);
+
+ pg_free(readbuf);
+
+ return (int) dsize;
+}
+
+/*
+ * Compress size bytes from ptr and write them to the stream.
+ */
+static size_t
+LZ4File_write(const void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ size_t status;
+ int remaining = size;
+
+ if (!fs->inited && LZ4File_init(fs, size, true))
+ return -1;
+
+ while (remaining > 0)
+ {
+ int chunk = remaining < LZ4_IN_SIZE ? remaining : LZ4_IN_SIZE;
+
+ remaining -= chunk;
+
+ status = LZ4F_compressUpdate(fs->ctx, fs->buffer, fs->buflen,
+ ptr, chunk, NULL);
+ if (LZ4F_isError(status))
+ {
+ fs->errcode = status;
+ return -1;
+ }
+
+ if (fwrite(fs->buffer, 1, status, fs->fp) != status)
+ {
+ errno = errno ? : ENOSPC;
+ return 1;
+ }
+ }
+
+ return size;
+}
+
+/*
+ * fread() equivalent implementation for LZ4 compressed files.
+ */
+static size_t
+LZ4File_read(void *ptr, size_t size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ int ret;
+
+ ret = LZ4File_read_internal(fs, ptr, size, false);
+ if (ret != size && !LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ return ret;
+}
+
+/*
+ * fgetc() equivalent implementation for LZ4 compressed files.
+ */
+static int
+LZ4File_getc(CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ unsigned char c;
+
+ if (LZ4File_read_internal(fs, &c, 1, false) != 1)
+ {
+ if (!LZ4File_eof(CFH))
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+
+ return c;
+}
+
+/*
+ * fgets() equivalent implementation for LZ4 compressed files.
+ */
+static char *
+LZ4File_gets(char *ptr, int size, CompressFileHandle *CFH)
+{
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ size_t dsize;
+
+ dsize = LZ4File_read_internal(fs, ptr, size, true);
+ if (dsize < 0)
+ pg_fatal("could not read from input file: %s", LZ4File_get_error(CFH));
+
+ /* Done reading */
+ if (dsize == 0)
+ return NULL;
+
+ return ptr;
+}
+
+/*
+ * Finalize (de)compression of a stream. When compressing it will write any
+ * remaining content and/or generated footer from the LZ4 API.
+ */
+static int
+LZ4File_close(CompressFileHandle *CFH)
+{
+ FILE *fp;
+ LZ4File *fs = (LZ4File *) CFH->private_data;
+ size_t status;
+ int ret;
+
+ fp = fs->fp;
+ if (fs->inited)
+ {
+ if (fs->compressing)
+ {
+ status = LZ4F_compressEnd(fs->ctx, fs->buffer, fs->buflen, NULL);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ else if ((ret = fwrite(fs->buffer, 1, status, fs->fp)) != status)
+ {
+ errno = errno ? : ENOSPC;
+ WRITE_ERROR_EXIT;
+ }
+
+ status = LZ4F_freeCompressionContext(fs->ctx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end compression: %s",
+ LZ4F_getErrorName(status));
+ }
+ else
+ {
+ status = LZ4F_freeDecompressionContext(fs->dtx);
+ if (LZ4F_isError(status))
+ pg_fatal("failed to end decompression: %s",
+ LZ4F_getErrorName(status));
+ pg_free(fs->overflowbuf);
+ }
+
+ pg_free(fs->buffer);
+ }
+
+ pg_free(fs);
+
+ return fclose(fp);
+}
+
+static int
+LZ4File_open(const char *path, int fd, const char *mode,
+ CompressFileHandle *CFH)
+{
+ FILE *fp;
+ LZ4File *lz4fp = (LZ4File *) CFH->private_data;
+
+ if (fd >= 0)
+ fp = fdopen(fd, mode);
+ else
+ fp = fopen(path, mode);
+ if (fp == NULL)
+ {
+ lz4fp->errcode = errno;
+ return 1;
+ }
+
+ lz4fp->fp = fp;
+
+ return 0;
+}
+
+static int
+LZ4File_open_write(const char *path, const char *mode, CompressFileHandle *CFH)
+{
+ char *fname;
+ int ret;
+
+ fname = psprintf("%s.lz4", path);
+ ret = CFH->open_func(fname, -1, mode, CFH);
+ pg_free(fname);
+
+ return ret;
+}
+
+void
+InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ LZ4File *lz4fp;
+
+ CFH->open_func = LZ4File_open;
+ CFH->open_write_func = LZ4File_open_write;
+ CFH->read_func = LZ4File_read;
+ CFH->write_func = LZ4File_write;
+ CFH->gets_func = LZ4File_gets;
+ CFH->getc_func = LZ4File_getc;
+ CFH->eof_func = LZ4File_eof;
+ CFH->close_func = LZ4File_close;
+ CFH->get_error_func = LZ4File_get_error;
+
+ CFH->compression_spec = compression_spec;
+ lz4fp = pg_malloc0(sizeof(*lz4fp));
+ if (CFH->compression_spec.level >= 0)
+ lz4fp->prefs.compressionLevel = CFH->compression_spec.level;
+
+ CFH->private_data = lz4fp;
+}
+#else /* USE_LZ4 */
+void
+InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "LZ4");
+}
+
+void
+InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec)
+{
+ pg_fatal("this build does not support compression with %s", "LZ4");
+}
+#endif /* USE_LZ4 */
diff --git a/src/bin/pg_dump/compress_lz4.h b/src/bin/pg_dump/compress_lz4.h
new file mode 100644
index 0000000000..74595db1b9
--- /dev/null
+++ b/src/bin/pg_dump/compress_lz4.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * compress_lz4.h
+ * Interface to compress_io.c routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/compress_lz4.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _COMPRESS_LZ4_H_
+#define _COMPRESS_LZ4_H_
+
+#include "compress_io.h"
+
+extern void InitCompressorLZ4(CompressorState *cs, const pg_compress_specification compression_spec);
+extern void InitCompressLZ4(CompressFileHandle *CFH, const pg_compress_specification compression_spec);
+
+#endif /* _COMPRESS_LZ4_H_ */
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 714e492c60..712e08aa02 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -3,6 +3,7 @@
pg_dump_common_sources = files(
'compress_io.c',
'compress_gzip.c',
+ 'compress_lz4.c',
'dumputils.c',
'parallel.c',
'pg_backup_archiver.c',
@@ -17,7 +18,7 @@ pg_dump_common_sources = files(
pg_dump_common = static_library('libpgdump_common',
pg_dump_common_sources,
c_pch: pch_postgres_fe_h,
- dependencies: [frontend_code, libpq, zlib],
+ dependencies: [frontend_code, libpq, lz4, zlib],
kwargs: internal_lib_args,
)
@@ -85,7 +86,10 @@ tests += {
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'tap': {
- 'env': {'GZIP_PROGRAM': gzip.path()},
+ 'env': {
+ 'GZIP_PROGRAM': gzip.path(),
+ 'LZ4': program_lz4.found() ? program_lz4.path() : '',
+ },
'tests': [
't/001_basic.pl',
't/002_pg_dump.pl',
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f207c6f4d..119b7f2553 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -395,6 +395,10 @@ RestoreArchive(Archive *AHX)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
supports_compression = false;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ supports_compression = false;
#endif
if (supports_compression == false)
pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
@@ -2074,7 +2078,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
/*
* Check if the specified archive is a directory. If so, check if
- * there's a "toc.dat" (or "toc.dat.gz") file in it.
+ * there's a "toc.dat" (or "toc.dat.{gz,lz4}") file in it.
*/
if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
{
@@ -2084,6 +2088,10 @@ _discoverArchiveFormat(ArchiveHandle *AH)
#ifdef HAVE_LIBZ
if (_fileExistsInDirectory(AH->fSpec, "toc.dat.gz"))
return AH->format;
+#endif
+#ifdef USE_LZ4
+ if (_fileExistsInDirectory(AH->fSpec, "toc.dat.lz4"))
+ return AH->format;
#endif
pg_fatal("directory \"%s\" does not appear to be a valid archive (\"toc.dat\" does not exist)",
AH->fSpec);
@@ -3747,6 +3755,10 @@ ReadHead(ArchiveHandle *AH)
#ifndef HAVE_LIBZ
if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
unsupported = true;
+#endif
+#ifndef USE_LZ4
+ if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
+ unsupported = true;
#endif
if (unsupported)
pg_fatal("archive is compressed, but this installation does not support compression");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 44d957c038..1bb874b8e3 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -715,13 +715,12 @@ main(int argc, char **argv)
case PG_COMPRESSION_NONE:
/* fallthrough */
case PG_COMPRESSION_GZIP:
+ /* fallthrough */
+ case PG_COMPRESSION_LZ4:
break;
case PG_COMPRESSION_ZSTD:
pg_fatal("compression with %s is not yet supported", "ZSTD");
break;
- case PG_COMPRESSION_LZ4:
- pg_fatal("compression with %s is not yet supported", "LZ4");
- break;
}
/*
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 1110ffa5e2..d4b9b3652d 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -139,6 +139,80 @@ my %pgdump_runs = (
args => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
},
},
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_custom => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=custom',
+ '--compress=lz4', "--file=$tempdir/compression_lz4_custom.dump",
+ 'postgres',
+ ],
+ restore_cmd => [
+ 'pg_restore',
+ "--file=$tempdir/compression_lz4_custom.sql",
+ "$tempdir/compression_lz4_custom.dump",
+ ],
+ command_like => {
+ command => [
+ 'pg_restore',
+ '-l', "$tempdir/compression_lz4_custom.dump",
+ ],
+ expected => qr/Compression: lz4/,
+ name => 'data content is lz4 compressed'
+ },
+ },
+
+ # Do not use --no-sync to give test coverage for data sync.
+ compression_lz4_dir => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--jobs=2',
+ '--format=directory', '--compress=lz4:1',
+ "--file=$tempdir/compression_lz4_dir", 'postgres',
+ ],
+ # Give coverage for manually compressed blob.toc files during
+ # restore.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-z', '-f', '--rm',
+ "$tempdir/compression_lz4_dir/blobs.toc",
+ "$tempdir/compression_lz4_dir/blobs.toc.lz4",
+ ],
+ },
+ # Verify that data files where compressed
+ glob_patterns => [
+ "$tempdir/compression_lz4_dir/toc.dat",
+ "$tempdir/compression_lz4_dir/*.dat.lz4",
+ ],
+ restore_cmd => [
+ 'pg_restore', '--jobs=2',
+ "--file=$tempdir/compression_lz4_dir.sql",
+ "$tempdir/compression_lz4_dir",
+ ],
+ },
+
+ compression_lz4_plain => {
+ test_key => 'compression',
+ compile_option => 'lz4',
+ dump_cmd => [
+ 'pg_dump', '--format=plain', '--compress=lz4',
+ "--file=$tempdir/compression_lz4_plain.sql.lz4", 'postgres',
+ ],
+ # Decompress the generated file to run through the tests.
+ compress_cmd => {
+ program => $ENV{'LZ4'},
+ args => [
+ '-d', '-f',
+ "$tempdir/compression_lz4_plain.sql.lz4",
+ "$tempdir/compression_lz4_plain.sql",
+ ],
+ },
+ },
+
clean => {
dump_cmd => [
'pg_dump',
@@ -4175,11 +4249,11 @@ foreach my $run (sort keys %pgdump_runs)
my $run_db = 'postgres';
# Skip command-level tests for gzip if there is no support for it.
- if ( defined($pgdump_runs{$run}->{compile_option})
- && $pgdump_runs{$run}->{compile_option} eq 'gzip'
- && !$supports_gzip)
+ if ($pgdump_runs{$run}->{compile_option} &&
+ ($pgdump_runs{$run}->{compile_option} eq 'gzip' && !$supports_gzip) ||
+ ($pgdump_runs{$run}->{compile_option} eq 'lz4' && !$supports_lz4))
{
- note "$run: skipped due to no gzip support";
+ note "$run: skipped due to no $pgdump_runs{$run}->{compile_option} support";
next;
}
diff --git a/src/tools/pginclude/cpluspluscheck b/src/tools/pginclude/cpluspluscheck
index 8805237edb..d44ce64fd7 100755
--- a/src/tools/pginclude/cpluspluscheck
+++ b/src/tools/pginclude/cpluspluscheck
@@ -152,6 +152,7 @@ do
# as field names, which is unfortunate but we won't change it now.
test "$f" = src/bin/pg_dump/compress_gzip.h && continue
test "$f" = src/bin/pg_dump/compress_io.h && continue
+ test "$f" = src/bin/pg_dump/compress_lz4.h && continue
test "$f" = src/bin/pg_dump/parallel.h && continue
test "$f" = src/bin/pg_dump/pg_backup_archiver.h && continue
test "$f" = src/bin/pg_dump/pg_dump.h && continue
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 81a451641a..0e25c7f58a 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1381,6 +1381,7 @@ LWLock
LWLockHandle
LWLockMode
LWLockPadded
+LZ4CompressorState
LZ4F_compressionContext_t
LZ4F_decompressOptions_t
LZ4F_decompressionContext_t
--
2.34.1
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-19 17:27 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-20 11:19 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-20 15:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-21 10:09 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-12-22 17:08 ` Justin Pryzby <[email protected]>
0 siblings, 0 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-12-22 17:08 UTC (permalink / raw)
To: [email protected]; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
There's a couple of lz4 bits which shouldn't be present in 002: file
extension and comments.
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
@ 2022-12-19 17:42 ` Justin Pryzby <[email protected]>
1 sibling, 0 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-12-19 17:42 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: [email protected]; [email protected]; Rachel Heaton <[email protected]>
On Mon, Dec 19, 2022 at 01:06:00PM +0900, Michael Paquier wrote:
> On Sat, Dec 17, 2022 at 05:26:15PM -0600, Justin Pryzby wrote:
> > 001: still refers to "gzip", which is correct for -Fp and -Fd but not
> > for -Fc, for which it's more correct to say "zlib".
>
> Or should we begin by changing all these existing "not built with zlib
> support" error strings to the more generic "this build does not
> support compression with %s" to reduce the number of messages to
> translate? That would bring consistency with the other tools dealing
> with compression.
That's fine, but it doesn't touch on the issue I'm talking about, which
is that zlib != gzip.
BTW I noticed that that also affects the pg_dump file itself; 002
changes the file format to say "gzip", but that's wrong for -Fc, which
does not use gzip headers, which could be surprising to someone who
specified "gzip".
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
* Re: Add LZ4 compression in pg_dump
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
@ 2022-11-23 20:52 ` Justin Pryzby <[email protected]>
1 sibling, 0 replies; 67+ messages in thread
From: Justin Pryzby @ 2022-11-23 20:52 UTC (permalink / raw)
To: [email protected]; +Cc: Michael Paquier <[email protected]>; [email protected]; Rachel Heaton <[email protected]>
On Tue, Nov 22, 2022 at 10:00:47AM +0000, [email protected] wrote:
> For the record I am currently working on it simply unsure if I should submit
> WIP patches and add noise to the list or wait until it is in a state that I
> feel that the comments have been addressed.
>
> A new version that I feel that is in a decent enough state for review should
> be ready within this week. I am happy to drop the patch if you think I should
> not work on it though.
I hope you'll want to continue work on it. The patch record is like a
request for review, so it's closed if there's nothing ready to review.
I think you should re-send patches (and update the CF app) as often as
they're ready for more review. Your 001 commit (which is almost the
same as what I wrote 2 years ago) still needs to account for some review
comments, and the whole patch set ought to pass cirrusci tests. At that
point, you'll be ready for another round of review, even if there's
known TODO/FIXME items in later patches.
BTW I saw that you updated your branch on github. You'll need to make
the corresponding changes to ./meson.build that you made to ./Makefile.
https://wiki.postgresql.org/wiki/Meson_for_patch_authors
https://wiki.postgresql.org/wiki/Meson
--
Justin
^ permalink raw reply [nested|flat] 67+ messages in thread
end of thread, other threads:[~2025-10-14 05:40 UTC | newest]
Thread overview: 67+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-05-30 13:11 [PATCH] Shrink GISTSTATE Andreas Karlsson <[email protected]>
2022-03-04 16:10 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 05:20 ` Re: Add LZ4 compression in pg_dump Greg Stark <[email protected]>
2022-03-25 13:22 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-25 23:13 ` Re: Add LZ4 compression in pg_dump Rachel Heaton <[email protected]>
2022-03-25 23:43 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-26 05:57 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 06:14 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-29 07:27 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 09:46 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-30 05:54 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-30 15:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-03-31 02:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-04-01 15:06 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-04-05 01:34 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-04-05 10:55 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2025-10-12 17:24 ` Re: Add LZ4 compression in pg_dump Tom Lane <[email protected]>
2025-10-14 05:40 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-26 16:21 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-26 18:33 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-26 23:37 ` Re: Add LZ4 compression in pg_dump Daniel Gustafsson <[email protected]>
2022-03-26 23:51 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-27 19:55 ` Re: Add LZ4 compression in pg_dump Daniel Gustafsson <[email protected]>
2022-03-27 14:13 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-27 16:06 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-03-28 12:36 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-29 05:03 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-03-29 13:14 ` Re: Add LZ4 compression in pg_dump Robert Haas <[email protected]>
2022-03-30 06:36 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-06-26 15:55 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-06-28 08:14 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-07-05 13:22 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-07-05 15:13 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-07-21 05:04 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-08-02 18:42 ` Re: Add LZ4 compression in pg_dump Jacob Champion <[email protected]>
2022-08-05 14:23 ` Re: Add LZ4 compression in pg_dump Georgios Kokolatos <[email protected]>
2022-11-02 13:28 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-06 14:53 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-20 17:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-20 23:13 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-22 10:00 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-22 10:49 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-28 16:32 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-29 06:19 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 07:08 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-29 12:10 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-11-30 00:50 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-11-30 17:11 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-01 02:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-01 14:58 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-02 01:56 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-02 16:15 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-03 02:45 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 07:05 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-05 12:48 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-06 00:22 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-06 15:52 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-17 23:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 04:06 ` Re: Add LZ4 compression in pg_dump Michael Paquier <[email protected]>
2022-12-19 17:03 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-19 17:27 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-20 11:19 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-20 15:26 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-21 10:09 ` Re: Add LZ4 compression in pg_dump [email protected]
2022-12-22 17:08 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-12-19 17:42 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
2022-11-23 20:52 ` Re: Add LZ4 compression in pg_dump Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox