public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 03/20] Support multiple compression algs/levels/opts..
31+ messages / 5 participants
[nested] [flat]
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 03/20] Support multiple compression algs/levels/opts..
@ 2020-12-22 06:23 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw)
The existing implementation abtracts compressed and noncompressed I/O.
This preliminary commit is intended to also allow for multiple compression
algorithms.
---
src/bin/pg_dump/compress_io.c | 220 +++++++++++---------------
src/bin/pg_dump/compress_io.h | 19 +--
src/bin/pg_dump/pg_backup.h | 23 ++-
src/bin/pg_dump/pg_backup_archiver.c | 45 +++---
src/bin/pg_dump/pg_backup_archiver.h | 12 +-
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 17 +-
src/bin/pg_dump/pg_backup_tar.c | 33 ++--
src/bin/pg_dump/pg_dump.c | 115 +++++++++++---
9 files changed, 272 insertions(+), 218 deletions(-)
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 6a428978d4..db16fd33f2 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -74,12 +74,9 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
-static void InitCompressorZlib(CompressorState *cs, int level);
+static void InitCompressorZlib(CompressorState *cs, Compress *compress);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
@@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(Compress *compression, WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
-
-#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- fatal("not built with zlib support");
-#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->comprAlg = compression->alg;
/*
* Perform compression algorithm specific initialization.
*/
+ Assert (compression->alg != COMPR_ALG_DEFAULT);
+ switch (compression->alg)
+ {
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ case COMPR_ALG_LIBZ:
+ InitCompressorZlib(cs, compression);
+ break;
#endif
+ case COMPR_ALG_NONE:
+ /* Do nothing */
+ break;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
return cs;
}
@@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (AH->compression.alg)
{
+ case COMPR_ALG_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
ReadDataFromArchiveZlib(AH, readF);
-#else
- fatal("not built with zlib support");
+ break;
#endif
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
{
switch (cs->comprAlg)
{
- case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
+ case COMPR_ALG_LIBZ:
WriteDataToArchiveZlib(AH, cs, data, dLen);
-#else
- fatal("not built with zlib support");
-#endif
break;
+#endif
case COMPR_ALG_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
}
}
@@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs)
*/
static void
-InitCompressorZlib(CompressorState *cs, int level)
+InitCompressorZlib(CompressorState *cs, Compress *compress)
{
z_streamp zp;
@@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level)
cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
- if (deflateInit(zp, level) != Z_OK)
+ if (deflateInit(zp, compress->level) != Z_OK)
fatal("could not initialize compression library: %s",
zp->msg);
@@ -424,9 +401,7 @@ struct cfp
#endif
};
-#ifdef HAVE_LIBZ
static int hasSuffix(const char *filename, const char *suffix);
-#endif
/* free() without changing errno; useful in several places below */
static void
@@ -442,34 +417,31 @@ free_keep_errno(void *p)
* Open a file for reading. 'path' is the file to open, and 'mode' should
* be either "r" or "rb".
*
- * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path'
- * doesn't already have it) and try again. So if you pass "foo" as 'path',
+ * If the file at 'path' does not exist, we search with compressed suffix (if 'path'
+ * doesn't already have one) and try again. So if you pass "foo" as 'path',
* this will open either "foo" or "foo.gz".
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_read(const char *path, const char *mode)
+cfopen_read(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
-#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compression);
else
-#endif
{
- fp = cfopen(path, mode, 0);
-#ifdef HAVE_LIBZ
+ fp = cfopen(path, mode, compression);
if (fp == NULL)
{
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fname = psprintf("%s%s", path, suffix);
+ fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
}
-#endif
}
return fp;
}
@@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * Use compression if specified.
+ * The appropriate suffix is automatically added to 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode, Compress *compression)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compression->alg == COMPR_ALG_NONE)
+ fp = cfopen(path, mode, compression);
else
{
-#ifdef HAVE_LIBZ
char *fname;
+ const char *suffix = compress_suffix(compression);
- fname = psprintf("%s.gz", path);
+ fname = psprintf("%s%s", path, suffix);
fp = cfopen(fname, mode, compression);
free_keep_errno(fname);
-#else
- fatal("not built with zlib support");
- fp = NULL; /* keep compiler quiet */
-#endif
}
return fp;
}
@@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression)
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode, Compress *compression)
{
- cfp *fp = pg_malloc(sizeof(cfp));
+ cfp *fp = pg_malloc0(sizeof(cfp));
- if (compression != 0)
+ switch (compression->alg)
{
#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
+ case COMPR_ALG_LIBZ:
+ if (compression->level != Z_DEFAULT_COMPRESSION)
{
/* user has specified a compression level, so tell zlib to use it */
char mode_compression[32];
snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
+ mode, compression->level);
fp->compressedfp = gzopen(path, mode_compression);
}
else
@@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression)
fp->compressedfp = gzopen(path, mode);
}
- fp->uncompressedfp = NULL;
if (fp->compressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
-#else
- fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ return fp;
#endif
+
+ case COMPR_ALG_NONE:
fp->uncompressedfp = fopen(path, mode);
if (fp->uncompressedfp == NULL)
{
free_keep_errno(fp);
fp = NULL;
}
- }
+ return fp;
- return fp;
+ default:
+ /* Should not happen */
+ fatal("requested compression not available in this installation");
+ }
}
@@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp)
fatal("could not read from input file: %s",
errnum == Z_ERRNO ? strerror(errno) : errmsg);
}
+ return ret;
}
- else
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
+
+ ret = fread(ptr, 1, size, fp->uncompressedfp);
+ if (ret != size && !feof(fp->uncompressedfp))
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzwrite(fp->compressedfp, ptr, size);
- else
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ return fwrite(ptr, 1, size, fp->uncompressedfp);
}
int
@@ -622,15 +585,12 @@ cfgetc(cfp *fp)
else
fatal("could not read from input file: end of file");
}
+ return ret;
}
- else
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
- }
-
+ ret = fgetc(fp->uncompressedfp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->uncompressedfp);
return ret;
}
@@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzgets(fp->compressedfp, buf, len);
- else
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ return fgets(buf, len, fp->uncompressedfp);
}
int
@@ -660,15 +619,13 @@ cfclose(cfp *fp)
{
result = gzclose(fp->compressedfp);
fp->compressedfp = NULL;
+ return result;
}
- else
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
- }
- free_keep_errno(fp);
+ result = fclose(fp->uncompressedfp);
+ fp->uncompressedfp = NULL;
+ free_keep_errno(fp);
return result;
}
@@ -678,9 +635,9 @@ cfeof(cfp *fp)
#ifdef HAVE_LIBZ
if (fp->compressedfp)
return gzeof(fp->compressedfp);
- else
#endif
- return feof(fp->uncompressedfp);
+
+ return feof(fp->uncompressedfp);
}
const char *
@@ -699,7 +656,6 @@ get_cfp_error(cfp *fp)
return strerror(errno);
}
-#ifdef HAVE_LIBZ
static int
hasSuffix(const char *filename, const char *suffix)
{
@@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix)
suffixlen) == 0;
}
-#endif
+/*
+ * Return a string for the given AH's compression.
+ * The string is statically allocated.
+ */
+const char *
+compress_suffix(Compress *compression)
+{
+ switch (compression->alg)
+ {
+ case COMPR_ALG_LIBZ:
+ return ".gz";
+ default:
+ return "";
+ }
+}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index d2e6e1b854..2c073676eb 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
-extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression);
+extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
@@ -67,4 +61,7 @@ extern int cfclose(cfp *fp);
extern int cfeof(cfp *fp);
extern const char *get_cfp_error(cfp *fp);
+/* also used by tar */
+extern const char * compress_suffix(Compress *compression);
+
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 9d0056a569..f2390b7937 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -72,6 +72,25 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+typedef enum
+{
+ COMPR_ALG_DEFAULT = -1,
+ COMPR_ALG_NONE,
+ COMPR_ALG_LIBZ,
+} CompressionAlgorithm;
+/* Should be called "method" or "library" ? */
+
+typedef struct Compress {
+ CompressionAlgorithm alg;
+ int level;
+ /* Is a nondefault level set ? This is useful since different compression
+ * methods have different "default" levels. For now we assume the levels
+ * are all integer, though.
+ */
+ bool level_set;
+} Compress;
+
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -125,7 +144,7 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ Compress compression;
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 1f82c6499b..3eb6c55600 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -70,7 +70,7 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
+static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress);
static OutputContext SaveOutput(ArchiveHandle *AH);
static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
@@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
@@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ Compress compress = {0};
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead,
+ setupRestoreWorker);
return (Archive *) AH;
}
@@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX)
* Make sure we won't need (de)compression we haven't got
*/
#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
@@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
+ SetOutput(AH, ropt->filename, &ropt->compression);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX)
char stamp_str[64];
sav = SaveOutput(AH);
+ Assert(ropt->compression.alg == COMPR_ALG_NONE);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, &ropt->compression);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
@@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX)
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount, AH->compression.alg);
switch (AH->format)
{
@@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression)
{
int fn;
@@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
/* If compression explicitly requested, use gzopen */
#ifdef HAVE_LIBZ
- if (compression != 0)
+ if (compression->alg != COMPR_ALG_NONE)
{
char fmode[14];
/* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
+ sprintf(fmode, "wb%d", compression->level);
if (fn >= 0)
AH->OF = gzdopen(dup(fn), fmode);
else
@@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ Compress *compression, bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
@@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compression = *compression;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
@@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compression.alg);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH)
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compression.alg = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compression.alg = ReadInt(AH);
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compression.alg = Z_DEFAULT_COMPRESSION;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index a8ea5c7eae..6e033d040e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -47,7 +47,8 @@
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
+/* this is just the redefinition of a libz constant, in case zlib isn't
+ * available */
#define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream
@@ -329,14 +330,7 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ Compress compression; /* Compression requested on open */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index 77d402c323..55a887a236 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 650b542fce..8bf869c6ca 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH)
setFilePath(AH, fname, "toc.dat");
- tocFH = cfopen_read(fname, PG_BINARY_R);
+ tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression);
if (tocFH == NULL)
fatal("could not open input file \"%s\": %m", fname);
@@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename)
if (!filename)
return;
- cfp = cfopen_read(filename, PG_BINARY_R);
+ cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression);
if (!cfp)
fatal("could not open input file \"%s\": %m", filename);
@@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH)
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
char line[MAXPGPATH];
+ Compress nocompression = {0};
StartRestoreBlobs(AH);
setFilePath(AH, fname, "blobs.toc");
- ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
+ ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open large object TOC file \"%s\" for input: %m",
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
{
cfp *tocFH;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "toc.dat");
@@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression);
if (tocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
char fname[MAXPGPATH];
+ Compress nocompression = {0};
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression);
if (ctx->blobsTocFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
}
@@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression);
if (ctx->dataFH == NULL)
fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 61c9c87a9f..4ba79ab924 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -39,6 +39,7 @@
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
#include "pg_backup_utils.h"
+#include "compress_io.h"
#include "pgtar.h"
static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
@@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
/*
* We don't support compression because reading the files back is not
- * possible since gzdopen uses buffered IO which totally screws file
+ * possible since gzdopen uses buffered IO which totally screws file XXX
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
fatal("compression is not supported by tar archive format");
}
else
@@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
if (te->dataDumper != NULL)
{
-#ifdef HAVE_LIBZ
- if (AH->compression == 0)
- sprintf(fn, "%d.dat", te->dumpId);
- else
- sprintf(fn, "%d.dat.gz", te->dumpId);
-#else
- sprintf(fn, "%d.dat", te->dumpId);
-#endif
+ const char *suffix = compress_suffix(&AH->compression);
+ sprintf(fn, "%d.dat%s", te->dumpId, suffix);
ctx->filename = pg_strdup(fn);
}
else
@@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression == 0)
+ if (AH->compression.alg == COMPR_ALG_NONE)
tm->nFH = ctx->tarFH;
else
fatal("compression is not supported by tar archive format");
@@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
#ifdef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
{
- sprintf(fmode, "wb%d", AH->compression);
+ sprintf(fmode, "wb%d", AH->compression.level);
tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
if (tm->zFH == NULL)
fatal("could not open temporary file");
@@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
/*
* Close the GZ file since we dup'd. This will flush the buffers.
*/
- if (AH->compression != 0)
+ if (AH->compression.alg != COMPR_ALG_NONE)
if (GZCLOSE(th->zFH) != 0)
fatal("could not close tar member");
@@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
+ ropt->compression.alg = COMPR_ALG_NONE;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
lclContext *ctx = (lclContext *) AH->formatData;
lclTocEntry *tctx = (lclTocEntry *) te->formatData;
char fname[255];
- char *sfx;
+ const char *sfx;
if (oid == 0)
fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
- sfx = ".gz";
- else
- sfx = "";
-
+ sfx = compress_suffix(&AH->compression);
sprintf(fname, "blob_%u.dat%s", oid, sfx);
tarPrintf(ctx->blobToc, "%u %s\n", oid, fname);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 1ab98a2286..4cbc79aedc 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
+#include "compress_io.h"
#include "pg_backup_db.h"
#include "pg_backup_utils.h"
#include "pg_dump.h"
@@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX);
static TableInfo *getRootTableInfo(TableInfo *tbinfo);
+/* Parse the string into compression options */
+static void
+parse_compression(const char *optarg, Compress *compress)
+{
+ if (optarg[0] == '0' && optarg[1] == '\0')
+ compress->alg = COMPR_ALG_NONE;
+ else if ((optarg[0] > '0' && optarg[0] <= '9') ||
+ optarg[0] == '-')
+ {
+ compress->alg = COMPR_ALG_LIBZ;
+ compress->level_set = true;
+ compress->level = atoi(optarg);
+ if (optarg[1] != '\0')
+ {
+ pg_log_error("compression level must be in range 0..9");
+ exit_nicely(1);
+ }
+ }
+ else
+ {
+ /* Parse a more flexible string like level=3 alg=zlib opts=long */
+ for (;;)
+ {
+ char *eq = strchr(optarg, '=');
+ int len;
+
+ if (eq == NULL)
+ {
+ pg_log_error("compression options must be key=value: %s", optarg);
+ exit_nicely(1);
+ }
+
+ len = eq - optarg;
+ if (strncmp(optarg, "alg", len) == 0)
+ {
+ if (strchr(eq, ' '))
+ len = strchr(eq, ' ') - eq - 1;
+ else
+ len = strlen(eq) - len;
+ if (strncmp(1+eq, "zlib", len) == 0 ||
+ strncmp(1+eq, "libz", len) == 0)
+ compress->alg = COMPR_ALG_LIBZ;
+ else
+ {
+ pg_log_error("unknown compression algorithm: %s", 1+eq);
+ exit_nicely(1);
+ }
+ }
+ else if (strncmp(optarg, "level", len) == 0)
+ {
+ compress->level = atoi(1+eq);
+ compress->level_set = true;
+ }
+ else
+ {
+ pg_log_error("unknown compression setting: %s", optarg);
+ exit_nicely(1);
+ }
+
+ optarg = strchr(eq, ' ');
+ if (!optarg++)
+ break;
+ }
+
+ if (!compress->level_set)
+ {
+ const int default_compress_level[] = {
+ 0, /* COMPR_ALG_NONE */
+ Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */
+ };
+
+ compress->level = default_compress_level[compress->alg];
+ }
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -319,7 +396,7 @@ main(int argc, char **argv)
char *use_role = NULL;
long rowsPerInsert;
int numWorkers = 1;
- int compressLevel = -1;
+ Compress compress = { .alg = COMPR_ALG_DEFAULT };
int plainText = 0;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -532,12 +609,7 @@ main(int argc, char **argv)
break;
case 'Z': /* Compression Level */
- compressLevel = atoi(optarg);
- if (compressLevel < 0 || compressLevel > 9)
- {
- pg_log_error("compression level must be in range 0..9");
- exit_nicely(1);
- }
+ parse_compression(optarg, &compress);
break;
case 0:
@@ -679,20 +751,28 @@ main(int argc, char **argv)
plainText = 1;
/* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ if (compress.alg == COMPR_ALG_DEFAULT)
{
-#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ {
+#ifdef HAVE_LIBZ
+ compress.alg = COMPR_ALG_LIBZ;
+ compress.level = Z_DEFAULT_COMPRESSION;
#endif
- compressLevel = 0;
+ }
+ else
+ {
+ compress.alg = COMPR_ALG_NONE;
+ compress.level = 0;
+ }
}
#ifndef HAVE_LIBZ
- if (compressLevel != 0)
+ if (compress.alg == COMPR_ALG_LIBZ)
+ {
pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
+ compress.alg = 0;
+ }
#endif
/*
@@ -723,7 +803,7 @@ main(int argc, char **argv)
fatal("option --index-collation-versions-unknown only works in binary upgrade mode");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
+ fout = CreateArchive(filename, archiveFormat, &compress, dosync,
archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
@@ -957,10 +1037,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compression = compress;
ropt->suppressDumpWarnings = true; /* We've already shown them */
--
2.17.0
--GRPZ8SYKNexpdSJ7
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0004-struct-compressLibs.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH] Clean up property graph error messages
@ 2026-05-04 19:57 Ayush Tiwari <[email protected]>
2026-05-05 02:50 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
2026-05-05 07:43 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
2026-05-07 08:45 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
0 siblings, 3 replies; 31+ messages in thread
From: Ayush Tiwari @ 2026-05-04 19:57 UTC (permalink / raw)
To: pgsql-hackers; Peter Eisentraut <[email protected]>; Ashutosh Bapat <[email protected]>
Hi,
While looking at the SQL/PGQ property graph error paths, I noticed a
few small cleanups in propgraphcmds.c.
The attached patch fixes a user-visible error message from "mismatching
properties names" to "mismatching property names", and moves a
ReleaseSysCache() call before an ERROR ereport in
check_element_properties().
The existing code should be cleaned up by
the resource owner on the ERROR path, but the explicit ReleaseSysCache()
placed after ereport(ERROR) was unreachable.
The regression expected output is updated for the changed error text.
[On a separate note, it might be better to change all "vertexes" to
"vertices",
haven't included that in the patch though since the other one is not exactly
wrong?]
Regards,
Ayus
Attachments:
[application/octet-stream] v1-0001-Clean-up-property-graph-error-messages.patch (4.2K, ../../CAJTYsWXFy1j_T82+M_S9kFxU414tQYnZQD-b82=oL_LbG_5fPQ@mail.gmail.com/3-v1-0001-Clean-up-property-graph-error-messages.patch)
download | inline diff:
From 4c5aca2d5e754911969831a658d511fa53bdfe03 Mon Sep 17 00:00:00 2001
From: Ayush Tiwari <[email protected]>
Date: Tue, 5 May 2026 01:13:18 +0530
Subject: [PATCH v1] Clean up property graph error messages
check_element_properties() saved a ReleaseSysCache() call until after
ereport(ERROR), making the explicit release unreachable. Copy the
catalog fields needed for the error message and release the syscache
entry before building and reporting the error.
Also fix the wording of a property graph label consistency error from
"mismatching properties names" to "mismatching property names", and
update the matching expected output.
---
src/backend/commands/propgraphcmds.c | 14 +++++++++-----
.../regress/expected/create_property_graph.out | 4 ++--
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/src/backend/commands/propgraphcmds.c b/src/backend/commands/propgraphcmds.c
index 6d509fccf46..7784dd0f5c5 100644
--- a/src/backend/commands/propgraphcmds.c
+++ b/src/backend/commands/propgraphcmds.c
@@ -1113,6 +1113,8 @@ check_element_properties(Oid peoid)
HeapTuple tuple3;
Form_pg_propgraph_element elform;
List *dpcontext;
+ char *element_name;
+ Oid element_relid;
char *dpa,
*dpb;
@@ -1120,7 +1122,11 @@ check_element_properties(Oid peoid)
if (!tuple3)
elog(ERROR, "cache lookup failed for property graph element %u", peoid);
elform = (Form_pg_propgraph_element) GETSTRUCT(tuple3);
- dpcontext = deparse_context_for(get_rel_name(elform->pgerelid), elform->pgerelid);
+ element_name = pstrdup(NameStr(elform->pgealias));
+ element_relid = elform->pgerelid;
+ ReleaseSysCache(tuple3);
+
+ dpcontext = deparse_context_for(get_rel_name(element_relid), element_relid);
dpa = deparse_expression(na, dpcontext, false, false);
dpb = deparse_expression(nb, dpcontext, false, false);
@@ -1141,10 +1147,8 @@ check_element_properties(Oid peoid)
ereport(ERROR,
errcode(ERRCODE_SYNTAX_ERROR),
errmsg("element \"%s\" property \"%s\" expression mismatch: %s vs. %s",
- NameStr(elform->pgealias), get_propgraph_property_name(propoid), dpa, dpb),
+ element_name, get_propgraph_property_name(propoid), dpa, dpb),
errdetail("In a property graph element, a property of the same name has to have the same expression in each label."));
-
- ReleaseSysCache(tuple3);
}
break;
@@ -1266,7 +1270,7 @@ check_element_label_properties(Oid ellabeloid)
if (diff1 || diff2)
ereport(ERROR,
errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
- errmsg("mismatching properties names in definition of label \"%s\"", get_propgraph_label_name(labelid)));
+ errmsg("mismatching property names in definition of label \"%s\"", get_propgraph_label_name(labelid)));
}
/*
diff --git a/src/test/regress/expected/create_property_graph.out b/src/test/regress/expected/create_property_graph.out
index bc9a596ec89..bfa6712b365 100644
--- a/src/test/regress/expected/create_property_graph.out
+++ b/src/test/regress/expected/create_property_graph.out
@@ -210,11 +210,11 @@ CREATE PROPERTY GRAPH gx
t1 KEY (a) LABEL l1 PROPERTIES (a, b),
t2 KEY (i) LABEL l1 PROPERTIES (i AS a, j AS j) -- mismatching property names on label
);
-ERROR: mismatching properties names in definition of label "l1"
+ERROR: mismatching property names in definition of label "l1"
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS yy, b AS zz); -- mismatching number of properties on label
ERROR: mismatching number of properties in definition of label "t3l1"
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS zz); -- mismatching property names on label
-ERROR: mismatching properties names in definition of label "t3l1"
+ERROR: mismatching property names in definition of label "t3l1"
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x); -- mismatching number of properties on label
ERROR: mismatching number of properties in definition of label "t3l1"
ALTER PROPERTY GRAPH g1 OWNER TO regress_graph_user1;
--
2.34.1
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
@ 2026-05-05 02:50 ` Chao Li <[email protected]>
2026-05-06 06:23 ` Re: [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2 siblings, 1 reply; 31+ messages in thread
From: Chao Li @ 2026-05-05 02:50 UTC (permalink / raw)
To: Ayush Tiwari <[email protected]>; +Cc: pgsql-hackers; Peter Eisentraut <[email protected]>; Ashutosh Bapat <[email protected]>
> On May 5, 2026, at 03:57, Ayush Tiwari <[email protected]> wrote:
>
> Hi,
>
> While looking at the SQL/PGQ property graph error paths, I noticed a
> few small cleanups in propgraphcmds.c.
>
> The attached patch fixes a user-visible error message from "mismatching
> properties names" to "mismatching property names”,
I think changing “properties names” to “property names” is correct.
I wonder if we can also change “mismatching” to “mismatched”, so the error message is:
```
"mismatched property names in definition of label \"%s\""
```
> and moves a
> ReleaseSysCache() call before an ERROR ereport in
> check_element_properties().
>
> The existing code should be cleaned up by
> the resource owner on the ERROR path, but the explicit ReleaseSysCache()
> placed after ereport(ERROR) was unreachable.
>
Agreed.
> The regression expected output is updated for the changed error text.
>
> [On a separate note, it might be better to change all "vertexes" to "vertices",
> haven't included that in the patch though since the other one is not exactly
> wrong?]
>
> Regards,
> Ayus
> <v1-0001-Clean-up-property-graph-error-messages.patch>
So, v1 LGTM. Only thing is that I think we can also make “mismatching” to “mismatched” in the errmsg.
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-05 02:50 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
@ 2026-05-06 06:23 ` Ayush Tiwari <[email protected]>
2026-05-06 06:45 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
2026-05-07 08:46 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
0 siblings, 2 replies; 31+ messages in thread
From: Ayush Tiwari @ 2026-05-06 06:23 UTC (permalink / raw)
To: Chao Li <[email protected]>; +Cc: pgsql-hackers; Peter Eisentraut <[email protected]>; Ashutosh Bapat <[email protected]>
Hi,
On Tue, 5 May 2026 at 08:20, Chao Li <[email protected]> wrote:
>
> So, v1 LGTM. Only thing is that I think we can also make “mismatching” to
> “mismatched” in the errmsg.
>
>
I agree that "mismatched property names" reads better, so I have updated
that in v2. I did not include the vertexes/vertices change here, since
Peter has already fixed that separately.
Regards,
Ayush
Attachments:
[application/octet-stream] v2-0001-Clean-up-property-graph-error-messages.patch (5.6K, ../../CAJTYsWVshTGyFRCMKnjiRgCnky1FB9vD-mjBHYGWDXppSdO0jQ@mail.gmail.com/3-v2-0001-Clean-up-property-graph-error-messages.patch)
download | inline diff:
From 5a28c695ec9218d53a0f9b913bc9d31067104d8c Mon Sep 17 00:00:00 2001
From: Ayush Tiwari <[email protected]>
Date: Wed, 6 May 2026 06:07:12 +0000
Subject: [PATCH v2] Clean up property graph error messages
Fix a typo in the error message for mismatched property names in
property graph labels.
Also move an unreachable ReleaseSysCache() before ereport(ERROR) in
check_element_properties(), copying the element alias and relation OID first
so the error path does not reference the released syscache tuple.
---
src/backend/commands/propgraphcmds.c | 14 +++++++++-----
.../regress/expected/create_property_graph.out | 8 ++++----
src/test/regress/sql/create_property_graph.sql | 4 ++--
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/src/backend/commands/propgraphcmds.c b/src/backend/commands/propgraphcmds.c
index 48284b84db9..6a35ba698ee 100644
--- a/src/backend/commands/propgraphcmds.c
+++ b/src/backend/commands/propgraphcmds.c
@@ -1113,6 +1113,8 @@ check_element_properties(Oid peoid)
HeapTuple tuple3;
Form_pg_propgraph_element elform;
List *dpcontext;
+ char *element_name;
+ Oid element_relid;
char *dpa,
*dpb;
@@ -1120,7 +1122,11 @@ check_element_properties(Oid peoid)
if (!tuple3)
elog(ERROR, "cache lookup failed for property graph element %u", peoid);
elform = (Form_pg_propgraph_element) GETSTRUCT(tuple3);
- dpcontext = deparse_context_for(get_rel_name(elform->pgerelid), elform->pgerelid);
+ element_name = pstrdup(NameStr(elform->pgealias));
+ element_relid = elform->pgerelid;
+ ReleaseSysCache(tuple3);
+
+ dpcontext = deparse_context_for(get_rel_name(element_relid), element_relid);
dpa = deparse_expression(na, dpcontext, false, false);
dpb = deparse_expression(nb, dpcontext, false, false);
@@ -1141,10 +1147,8 @@ check_element_properties(Oid peoid)
ereport(ERROR,
errcode(ERRCODE_SYNTAX_ERROR),
errmsg("element \"%s\" property \"%s\" expression mismatch: %s vs. %s",
- NameStr(elform->pgealias), get_propgraph_property_name(propoid), dpa, dpb),
+ element_name, get_propgraph_property_name(propoid), dpa, dpb),
errdetail("In a property graph element, a property of the same name has to have the same expression in each label."));
-
- ReleaseSysCache(tuple3);
}
break;
@@ -1266,7 +1270,7 @@ check_element_label_properties(Oid ellabeloid)
if (diff1 || diff2)
ereport(ERROR,
errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
- errmsg("mismatching properties names in definition of label \"%s\"", get_propgraph_label_name(labelid)));
+ errmsg("mismatched property names in definition of label \"%s\"", get_propgraph_label_name(labelid)));
}
/*
diff --git a/src/test/regress/expected/create_property_graph.out b/src/test/regress/expected/create_property_graph.out
index 4fc4759f18e..ebb5f70787b 100644
--- a/src/test/regress/expected/create_property_graph.out
+++ b/src/test/regress/expected/create_property_graph.out
@@ -208,13 +208,13 @@ ERROR: mismatching number of properties in definition of label "l1"
CREATE PROPERTY GRAPH gx
VERTEX TABLES (
t1 KEY (a) LABEL l1 PROPERTIES (a, b),
- t2 KEY (i) LABEL l1 PROPERTIES (i AS a, j AS j) -- mismatching property names on label
+ t2 KEY (i) LABEL l1 PROPERTIES (i AS a, j AS j) -- mismatched property names on label
);
-ERROR: mismatching properties names in definition of label "l1"
+ERROR: mismatched property names in definition of label "l1"
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS yy, b AS zz); -- mismatching number of properties on label
ERROR: mismatching number of properties in definition of label "t3l1"
-ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS zz); -- mismatching property names on label
-ERROR: mismatching properties names in definition of label "t3l1"
+ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS zz); -- mismatched property names on label
+ERROR: mismatched property names in definition of label "t3l1"
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x); -- mismatching number of properties on label
ERROR: mismatching number of properties in definition of label "t3l1"
ALTER PROPERTY GRAPH g1 OWNER TO regress_graph_user1;
diff --git a/src/test/regress/sql/create_property_graph.sql b/src/test/regress/sql/create_property_graph.sql
index 241f93df302..d1f93fe94a8 100644
--- a/src/test/regress/sql/create_property_graph.sql
+++ b/src/test/regress/sql/create_property_graph.sql
@@ -163,10 +163,10 @@ CREATE PROPERTY GRAPH gx
CREATE PROPERTY GRAPH gx
VERTEX TABLES (
t1 KEY (a) LABEL l1 PROPERTIES (a, b),
- t2 KEY (i) LABEL l1 PROPERTIES (i AS a, j AS j) -- mismatching property names on label
+ t2 KEY (i) LABEL l1 PROPERTIES (i AS a, j AS j) -- mismatched property names on label
);
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS yy, b AS zz); -- mismatching number of properties on label
-ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS zz); -- mismatching property names on label
+ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x, b AS zz); -- mismatched property names on label
ALTER PROPERTY GRAPH g4 ALTER VERTEX TABLE t1 ADD LABEL t3l1 PROPERTIES (a AS x); -- mismatching number of properties on label
ALTER PROPERTY GRAPH g1 OWNER TO regress_graph_user1;
--
2.43.0
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-05 02:50 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
2026-05-06 06:23 ` Re: [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
@ 2026-05-06 06:45 ` Chao Li <[email protected]>
1 sibling, 0 replies; 31+ messages in thread
From: Chao Li @ 2026-05-06 06:45 UTC (permalink / raw)
To: Ayush Tiwari <[email protected]>; +Cc: pgsql-hackers; Peter Eisentraut <[email protected]>; Ashutosh Bapat <[email protected]>
> On May 6, 2026, at 14:23, Ayush Tiwari <[email protected]> wrote:
>
> Hi,
>
>
> On Tue, 5 May 2026 at 08:20, Chao Li <[email protected]> wrote:
>
> So, v1 LGTM. Only thing is that I think we can also make “mismatching” to “mismatched” in the errmsg.
>
> I agree that "mismatched property names" reads better, so I have updated
> that in v2. I did not include the vertexes/vertices change here, since
> Peter has already fixed that separately.
>
> Regards,
> Ayush
> <v2-0001-Clean-up-property-graph-error-messages.patch>
V2 LGTM.
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-05 02:50 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
2026-05-06 06:23 ` Re: [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
@ 2026-05-07 08:46 ` Peter Eisentraut <[email protected]>
1 sibling, 0 replies; 31+ messages in thread
From: Peter Eisentraut @ 2026-05-07 08:46 UTC (permalink / raw)
To: Ayush Tiwari <[email protected]>; Chao Li <[email protected]>; +Cc: pgsql-hackers; Ashutosh Bapat <[email protected]>
On 06.05.26 08:23, Ayush Tiwari wrote:
> On Tue, 5 May 2026 at 08:20, Chao Li <[email protected]
> <mailto:[email protected]>> wrote:
>
>
> So, v1 LGTM. Only thing is that I think we can also make
> “mismatching” to “mismatched” in the errmsg.
>
> I agree that "mismatched property names" reads better, so I have updated
> that in v2.
I think the existing wording is better. In any case, you only changed a
few places and not others.
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
@ 2026-05-05 07:43 ` Peter Eisentraut <[email protected]>
2 siblings, 0 replies; 31+ messages in thread
From: Peter Eisentraut @ 2026-05-05 07:43 UTC (permalink / raw)
To: Ayush Tiwari <[email protected]>; pgsql-hackers; Ashutosh Bapat <[email protected]>
On 04.05.26 21:57, Ayush Tiwari wrote:
> [On a separate note, it might be better to change all "vertexes" to
> "vertices",
> haven't included that in the patch though since the other one is not exactly
> wrong?]
I have fixed that, thanks for pointing it out. We should stick to the
preferred terms.
(I'll look at the rest of your patch in a bit.)
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
@ 2026-05-07 08:45 ` Peter Eisentraut <[email protected]>
2026-05-07 09:04 ` Re: [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-11 06:39 ` Re: [PATCH] Clean up property graph error messages Ashutosh Bapat <[email protected]>
2 siblings, 2 replies; 31+ messages in thread
From: Peter Eisentraut @ 2026-05-07 08:45 UTC (permalink / raw)
To: Ayush Tiwari <[email protected]>; pgsql-hackers; Ashutosh Bapat <[email protected]>
On 04.05.26 21:57, Ayush Tiwari wrote:
> While looking at the SQL/PGQ property graph error paths, I noticed a
> few small cleanups in propgraphcmds.c.
>
> The attached patch fixes a user-visible error message from "mismatching
> properties names" to "mismatching property names",
I have fixed that.
> and moves a
> ReleaseSysCache() call before an ERROR ereport in
> check_element_properties().
>
> The existing code should be cleaned up by
> the resource owner on the ERROR path, but the explicit ReleaseSysCache()
> placed after ereport(ERROR) was unreachable.
I think that's fine. I don't think the change makes this better.
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-07 08:45 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
@ 2026-05-07 09:04 ` Ayush Tiwari <[email protected]>
1 sibling, 0 replies; 31+ messages in thread
From: Ayush Tiwari @ 2026-05-07 09:04 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: pgsql-hackers; Ashutosh Bapat <[email protected]>
Hi,
On Thu, 7 May 2026 at 14:15, Peter Eisentraut <[email protected]> wrote:
>
> > and moves a
> > ReleaseSysCache() call before an ERROR ereport in
> > check_element_properties().
> >
> > The existing code should be cleaned up by
> > the resource owner on the ERROR path, but the explicit ReleaseSysCache()
> > placed after ereport(ERROR) was unreachable.
>
> I think that's fine. I don't think the change makes this better.
>
IIUC that is dead code right now, it never reaches that point to release?
Regards,
Ayush
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: [PATCH] Clean up property graph error messages
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-07 08:45 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
@ 2026-05-11 06:39 ` Ashutosh Bapat <[email protected]>
1 sibling, 0 replies; 31+ messages in thread
From: Ashutosh Bapat @ 2026-05-11 06:39 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Ayush Tiwari <[email protected]>; pgsql-hackers
On Thu, May 7, 2026 at 2:15 PM Peter Eisentraut <[email protected]> wrote:
>
> On 04.05.26 21:57, Ayush Tiwari wrote:
> > While looking at the SQL/PGQ property graph error paths, I noticed a
> > few small cleanups in propgraphcmds.c.
> >
> > The attached patch fixes a user-visible error message from "mismatching
> > properties names" to "mismatching property names",
>
> I have fixed that.
>
> > and moves a
> > ReleaseSysCache() call before an ERROR ereport in
> > check_element_properties().
> >
> > The existing code should be cleaned up by
> > the resource owner on the ERROR path, but the explicit ReleaseSysCache()
> > placed after ereport(ERROR) was unreachable.
>
> I think that's fine. I don't think the change makes this better.
>
Yeah.
If we call ReleaseSysCache before ereport(), we need to add local
variables as you have done, which increases the lines of code. Since
ereport will never return, ReleaseSysCache() is not needed at all. But
leaving it there helps justifying fetching the attributes in the
ereport call. FWIW, it makes the code future proof, in a very rare
case if someone makes ereport conditional.
--
Best Wishes,
Ashutosh Bapat
^ permalink raw reply [nested|flat] 31+ messages in thread
end of thread, other threads:[~2026-05-11 06:39 UTC | newest]
Thread overview: 31+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <[email protected]>
2026-05-04 19:57 [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-05 02:50 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
2026-05-06 06:23 ` Re: [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-06 06:45 ` Re: [PATCH] Clean up property graph error messages Chao Li <[email protected]>
2026-05-07 08:46 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
2026-05-05 07:43 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
2026-05-07 08:45 ` Re: [PATCH] Clean up property graph error messages Peter Eisentraut <[email protected]>
2026-05-07 09:04 ` Re: [PATCH] Clean up property graph error messages Ayush Tiwari <[email protected]>
2026-05-11 06:39 ` Re: [PATCH] Clean up property graph error messages Ashutosh Bapat <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox